src/preprocessor/json_context_extractor.py

import json
import re
import os
import sys
from typing import Dict, List, Any
from collections import defaultdict
from fuzzywuzzy import fuzz  # For fuzzy deduplication; pip install fuzzywuzzy python-Levenshtein
import anthropic
from dotenv import load_dotenv

# Add the project root to the path
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
from database.supabase_manager import supabase_manager

# Load environment variables
load_dotenv()

class JSONContextExtractor:
    def __init__(self, chunk_size: int = 100, overlap_size: int = 20):
        self.chunk_size = chunk_size
        self.overlap_size = overlap_size
        
        # Initialize Anthropic client
        self.anthropic_client = anthropic.Anthropic(
            api_key=os.getenv("ANTHROPIC_API_KEY")
        )
        
        # Set default model name
        self.model_name = "claude-3-5-sonnet-20240620"
        
        # Test Supabase connection
        if not supabase_manager.is_connected():
            print("⚠️  Supabase connection failed. Operations will be logged only.")
            self.db_enabled = False
        else:
            print("✅ Supabase connection successful.")
            self.db_enabled = True

    def chunk_messages(self, messages: List[Dict[str, str]]) -> List[List[Dict[str, str]]]:
        """Split messages into overlapping chunks for LLM processing"""
        if not messages:
            return []
            
        chunks = []
        for i in range(0, len(messages), self.chunk_size - self.overlap_size):
            chunk = messages[i:i + self.chunk_size]
            if chunk:
                chunks.append(chunk)
            
            # Stop if we've reached the end
            if i + self.chunk_size >= len(messages):
                break
                
        return chunks

    def deduplicate_messages(self, messages: List[Dict]) -> List[Dict]:
        """Remove duplicate messages within a context using fuzzy matching"""
        seen = set()
        deduped = []
        for msg in messages:
            text = msg.get("message", "").strip().lower()
            if text:
                is_duplicate = any(fuzz.ratio(text, s) > 95 for s in seen)
                if not is_duplicate:
                    seen.add(text)
                    deduped.append(msg)
        return deduped

    def prepare_comprehensive_llm_prompt(self, chunk: List[Dict[str, str]], user_id: str) -> str:
        """Prepare prompt for comprehensive LLM extraction from message chunk with ownership validation"""
        deduped_chunk = self.deduplicate_messages(chunk)
        context_text = "\n".join([
            f"[{msg.get('id', i)}] {msg.get('sender', 'Unknown')}: {msg.get('message', '')}" 
            for i, msg in enumerate(deduped_chunk)
        ])
        
        prompt = f"""
You are an expert information extractor for a personal assistant memory system. Extract ONLY personal information that specifically belongs to or is claimed by the target user "{user_id}".

TARGET USER: {user_id}

CONVERSATION CHUNK:
{context_text}

CRITICAL OWNERSHIP VALIDATION RULES:
1. ONLY extract information that is specifically ABOUT {user_id} or claimed BY {user_id}
2. REJECT information about other people, temporary locations, or general mentions
3. REQUIRE clear ownership indicators like "my", "I am", "I live", "my name is", etc.
4. REJECT casual mentions that don't indicate personal ownership
5. If unclear who the information belongs to, DO NOT extract it

VALID OWNERSHIP PATTERNS:
✅ "{user_id} says: My address is..."
✅ "I live at..." (when {user_id} is speaking)
✅ "My phone number is..." (when {user_id} is speaking)  
✅ "I was born in..." (when {user_id} is speaking)
✅ "My wife/husband is..." (when {user_id} is speaking)

INVALID PATTERNS TO REJECT:
❌ "Let's meet at Mumbai" (temporary location, not personal address)
❌ "The office is in Bandra" (not personal address)
❌ "She lives in..." (someone else's information)
❌ Any location mentioned without clear personal ownership
❌ Business addresses, meeting locations, casual place mentions

MEMORY LAYERS TO EXTRACT (ONLY IF CLEARLY OWNED BY {user_id}):

LAYER 1 - Basic Personal Information (HIGHEST PRIORITY):
- Full names, ages, date of birth, nationality, gender, blood group (ONLY if about {user_id})
- Phone numbers, email addresses (ONLY if stated as THEIR contact info)
- Home addresses (ONLY if clearly stated as THEIR home/personal address)
- Relationship status, occupation, company details (ONLY if {user_id}'s info)
- Work location, car model, Work Address comes under LAYER 1

LAYER 2 - Document Information (HIGH PRIORITY):
- DONT EXTRACT INFORMATION IF IT IS NOT CLEARLY DESCRIBED. For example, user mentions about the aadhar card, but doesn't have any aadhar card ID/details, Then DONT EXTRACT
- Government IDs, certificates (ONLY if {user_id}'s documents)
- Document numbers, policies, licenses (ONLY if belonging to {user_id})
- Credit cards, bank details (ONLY if {user_id}'s financial info)

LAYER 3 - Relationships & Contacts (MEDIUM PRIORITY):
- Family members, friends, colleagues (ONLY if {user_id}'s relationships AND names are mentioned)
- CONTACT DETAILS OF FAMILY/FRIENDS (ONLY if {user_id} is sharing about THEIR contacts)
- IMPORTANT: Only extract relationship if SPECIFIC NAMES are mentioned
- REJECT: Generic "my wife", "my husband" without names
- ACCEPT: "my wife Sarah", "my husband John"

LAYER 4 - Preferences & Instructions (LOWER PRIORITY):
- Food preferences, allergies (ONLY if {user_id}'s preferences)
- Favorite places, vendors (ONLY if {user_id}'s preferences)
- Habits, routines (ONLY if {user_id}'s personal habits)
- DO NOT put travel plans here unless they are long-term preferences
- DO NOT CONSIDER EVERYTHING AS A PREFERENCE, UNTIL {user_id} SAYS OR LIKES IT. 

STRICT LAYER 3 RELATIONSHIP RULES:
- ONLY extract relationships if SPECIFIC NAMES are mentioned.
✅ "My wife Sarah" → Extract: relationship="spouse", name="Sarah"
✅ "My husband John works at..." → Extract: relationship="spouse", name="John"
✅ "My brother Mike lives in..." → Extract: relationship="brother", name="Mike"
❌ "My wife will come" → REJECT (no name mentioned)
❌ "Thanks guys" → REJECT (not a relationship statement)
❌ "I want to carry something for my 9 year old nephew" → REJECT (no name mentioned)


OWNERSHIP VALIDATION EXAMPLES:
✅ "My address is 402, Pinnacle Gold, Bandra" → Extract as {user_id}'s address
❌ "Let's meet at Pinnacle Gold, Bandra" → REJECT (not personal address)
✅ "I live in Mumbai" → Extract as {user_id}'s address  
❌ "Mumbai has good restaurants" → REJECT (general comment, not personal info)
✅ "My wife Sarah" → Extract as {user_id}'s spouse with name "Sarah"
❌ "My wife will come" → REJECT (no specific name mentioned)
❌ "Priya is coming" → REJECT (unclear relationship/ownership)

CRITICAL EXTRACTION RULES:
1. NEVER extract relationship information without specific names.
2. NEVER extract information from casual conversation phrases
3. ALWAYS ensure the information directly belongs to {user_id}
4. FOCUS ON Layer 1, 2, 3 - avoid putting travel plans in Layer 1
5. For relationships: ONLY extract if both relationship type AND name are clear
6. IF THE USER MENTIONS THE CONTACT OR DETAILS OF A FAMILY MEMBER/FRIEND, ONLY EXTRACT IF IT IS CLEAR THAT THE DETAILS BELONG TO {user_id}(For e.g., "My wife's number is..." then save it as spouse phone number)

CONFIDENCE SCORING BASED ON OWNERSHIP:
- 0.9-1.0: Direct personal claims with clear ownership ("My address is...")
- 0.8-0.9: Clear attribution to {user_id} ("I live at...")  
- Below 0.8: REJECT (unclear ownership)

RESPONSE FORMAT (JSON only):
{{
  "Layer1": [
    {{
      "detail": {{"type": "field_type", "value": "extracted_value"}},
      "confidence": 0.95,
      "evidence": [
        {{"message_id": "id", "message_snippet": "relevant_part"}}
      ],
      "timestamp": "2025-09-13 00:00:00",
      "ownership_reason": "Clear personal claim by {user_id}"
    }}
  ],
  "Layer2": [...],
  "Layer3": [...],
  "Layer4": [...]
}}

FIELD VALUE SPECIFICATIONS:
- relationship_status: Use "married" or "single". If mentions "wife"/"husband" use "married"
- gender: Use "Male" or "Female" 
- address: ONLY extract if clearly stated as personal/home address, NOT business/meeting locations or anything other than personal.
- spouse/spouse_name: Use ONLY if specific name is mentioned (e.g., "My wife Sarah" → spouse_name: "Sarah")
- family_member: Use ONLY if specific name is mentioned (e.g., "My brother Mike" → family_member: "Mike")
- phone_number: Use exact format from message,
- email: Use exact email address as written.

PROHIBITED EXTRACTIONS:
❌ Do NOT extract relationships (e.g., "wife", "husband", "nephew") as standalone values without names
❌ Do NOT extract meeting locations as personal addresses
❌ Do NOT extract third-party information as user's information

REMEMBER: When in doubt about ownership or clarity, DO NOT extract. It's better to miss information than to incorrectly attribute someone else's details to {user_id}.

Only return valid JSON. If no clearly owned information found for a layer, return empty array for that layer.
"""
        return prompt

    def call_llm_for_extraction(self, prompt: str) -> Dict:
        """Call Claude LLM for information extraction"""
        try:
            response = self.anthropic_client.messages.create(
                model=self.model_name,
                max_tokens=3000,  # Increased for comprehensive extraction
                temperature=0.1,
                messages=[{"role": "user", "content": prompt}]
            )
            
            # Extract text content from response
            response_text = response.content[0].text
            
            # Try to parse JSON response
            try:
                raw_data = json.loads(response_text)
                # Apply validation to filter out bad extractions
                validated_data = self._validate_extractions(raw_data)
                return validated_data
            except json.JSONDecodeError:
                print(f"Failed to parse LLM response as JSON: {response_text[:200]}...")
                return {}
                
        except Exception as e:
            print(f"LLM API error: {e}")
            return {}

    def _validate_extractions(self, raw_data: Dict) -> Dict:
        """Validate and filter out bad extractions"""
        validated_data = {}
        
        # Invalid values that should never be extracted
        invalid_relationship_values = {
            'wife', 'husband', 'nephew', 'niece', 'son', 'daughter', 'brother', 'sister',
            'mother', 'father', 'aunt', 'uncle', 'cousin', 'friend', 'colleague'
        }
        
        # Invalid evidence snippets that indicate bad extraction
        invalid_evidence_patterns = [
            'thanks guys', 'thank you', 'thanks', 'ok', 'okay', 'yes', 'no',
            'sure', 'great', 'perfect', 'sounds good', 'alright'
        ]
        
        for layer, nodes in raw_data.items():
            validated_nodes = []
            
            for node in nodes:
                if not isinstance(node, dict) or 'detail' not in node:
                    continue
                    
                detail = node['detail']
                fact_type = detail.get('type', '')
                fact_value = detail.get('value', '').lower().strip()
                evidence = node.get('evidence', [])
                
                # Skip invalid extractions
                skip_node = False
                
                # Check for invalid relationship extractions
                if fact_type in ['relationship', 'family_member', 'spouse'] and fact_value in invalid_relationship_values:
                    print(f"⚠️  Skipping invalid {fact_type}: '{fact_value}' (no specific name)")
                    skip_node = True
                
                # Check for invalid evidence snippets
                for ev in evidence:
                    snippet = ev.get('message_snippet', '').lower().strip()
                    if snippet in invalid_evidence_patterns:
                        print(f"⚠️  Skipping extraction from invalid evidence: '{snippet}'")
                        skip_node = True
                        break
                
                # Check for travel plans in Layer 1
                if layer == 'Layer1' and fact_type in ['travel_plan', 'flight_number', 'travel_date']:
                    print(f"⚠️  Moving {fact_type} from Layer1 to Layer4 (travel plans don't belong in basic info)")
                    # Move to Layer4 instead of rejecting
                    if 'Layer4' not in validated_data:
                        validated_data['Layer4'] = []
                    validated_data['Layer4'].append(node)
                    continue
                
                # Check confidence threshold
                confidence = node.get('confidence', 0)
                if confidence < 0.75:
                    print(f"⚠️  Skipping low confidence ({confidence}) extraction: {fact_type} = {fact_value}")
                    skip_node = True
                
                if not skip_node:
                    validated_nodes.append(node)
            
            if validated_nodes:
                validated_data[layer] = validated_nodes
        
        return validated_data

    async def store_in_db(self, extracted_nodes: Dict, user_id: str):
        """Store extracted nodes in Supabase DB using SupabaseManager
        Returns dict with newly_stored_nodes containing only facts that were actually added in this iteration"""
        if not self.db_enabled:
            print(f"\n=== EXTRACTED DATA FOR USER: {user_id} (DB DISABLED) ===")
            for layer, nodes in extracted_nodes.items():
                print(f"\n{layer}:")
                for i, node in enumerate(nodes, 1):
                    print(f"  {i}. Type: {node['detail']['type']}")
                    print(f"     Value: {node['detail']['value']}")
                    print(f"     Confidence: {node['confidence']}")
                    print(f"     Evidence: {len(node['evidence'])} message(s)")
            print("=" * 50)
            # Return all nodes as "newly stored" when DB is disabled (for testing), but filter empty layers
            filtered_extracted = {layer: nodes for layer, nodes in extracted_nodes.items() if nodes}
            return {"newly_stored_nodes": filtered_extracted, "stats": {"stored": 0, "duplicate": 0, "low_confidence_rejected": 0}}
        
        try:
            stored_nodes = 0
            duplicate_nodes = 0
            low_confidence_rejected = 0
            newly_stored_nodes = {}  # Track only nodes that were actually stored in this iteration
            
            # Minimum confidence threshold for ownership validation
            CONFIDENCE_THRESHOLD = 0.75
            
            for layer, nodes in extracted_nodes.items():
                layer_number = int(layer.replace("Layer", ""))
                newly_stored_nodes[layer] = []  # Initialize layer in newly stored nodes
                
                for node in nodes:
                    detail = node["detail"]
                    confidence = node["confidence"]
                    evidence = node["evidence"]
                    
                    # Apply confidence threshold - reject low confidence extractions
                    if confidence < CONFIDENCE_THRESHOLD:
                        low_confidence_rejected += 1
                        print(f"  ⚠️  Rejected low confidence {detail['type']}: {detail['value']} (confidence: {confidence:.2f})")
                        continue
                    
                    # Safely extract required fields
                    try:
                        fact_type = detail.get("type", "unknown")
                        fact_value = detail.get("value", "N/A")
                        
                        if fact_value == "N/A":
                            print(f"⚠️  Warning: Missing 'value' field in detail: {detail}")
                        
                        # Format concluded fact for human readability
                        concluded_fact = self._format_concluded_fact(user_id, fact_type, fact_value)
                        
                        # Store memory node directly in Supabase
                        node_id = await supabase_manager.store_memory_node(
                            user_phone=user_id,
                            layer=layer_number,
                            fact_type=fact_type,
                            content=fact_value,
                            concluded_fact=concluded_fact,
                            confidence=confidence,
                            evidence=evidence
                        )
                        
                        if node_id:
                            stored_nodes += 1
                            ownership_reason = node.get('ownership_reason', 'Ownership validated')
                            print(f"  ✅ Stored {fact_type}: {fact_value} (confidence: {confidence:.2f}) - {ownership_reason}")
                            # Add to newly stored nodes for this iteration
                            newly_stored_nodes[layer].append(node)
                        else:
                            duplicate_nodes += 1
                            print(f"  ⚠️  Failed to store {fact_type}: {fact_value}")
                            
                    except Exception as detail_error:
                        print(f"❌ Error processing detail: {str(detail_error)}")
                        print(f"❌ Detail structure: {detail}")
                        continue  # Skip this detail and continue with next one
            
            # Print summary
            print(f"\n📊 Database Summary for {user_id}:")
            print(f"   Stored: {stored_nodes} new nodes")
            print(f"   Failed: {duplicate_nodes} nodes")
            print(f"   Rejected (low confidence): {low_confidence_rejected} nodes")
            
            # Get user summary from database
            import asyncio
            summary = await supabase_manager.get_user_summary(user_id)
            if summary:
                print(f"   Total nodes in DB: {summary.get('total_nodes', 0)}")
                print(f"   By status: Approved={summary.get('approved_nodes', 0)}, Pending={summary.get('pending_nodes', 0)}, Rejected={summary.get('rejected_nodes', 0)}")
                if 'layers' in summary:
                    print(f"   By layers: {summary['layers']}")
            
            # Return only the newly stored nodes from this iteration
            # Filter out empty layers for cleaner UI display
            filtered_newly_stored = {layer: nodes for layer, nodes in newly_stored_nodes.items() if nodes}
            
            return {
                "newly_stored_nodes": filtered_newly_stored,
                "stats": {
                    "stored": stored_nodes,
                    "duplicate": duplicate_nodes,
                    "low_confidence_rejected": low_confidence_rejected
                }
            }
            
        except Exception as e:
            print(f"❌ Database error: {e}")
            # Fallback to logging
            print(f"\n=== EXTRACTED DATA FOR USER: {user_id} (DB ERROR FALLBACK) ===")
            for layer, nodes in extracted_nodes.items():
                print(f"\n{layer}:")
                for i, node in enumerate(nodes, 1):
                    print(f"  {i}. Type: {node['detail']['type']}")
                    print(f"     Value: {node['detail']['value']}")
                    print(f"     Confidence: {node['confidence']}")
                    print(f"     Evidence: {len(node['evidence'])} message(s)")
            print("=" * 50)
            # Return all nodes as fallback when DB has errors, but filter empty layers
            filtered_extracted = {layer: nodes for layer, nodes in extracted_nodes.items() if nodes}
            return {"newly_stored_nodes": filtered_extracted, "stats": {"stored": 0, "duplicate": 0, "low_confidence_rejected": 0}}

    def _format_concluded_fact(self, user_id: str, fact_type: str, value: str) -> str:
        """Format a concluded fact in human-readable form"""
        
        # Common fact type mappings with ownership emphasis
        fact_formatters = {
            'phone': lambda u, v: f"📱 Phone number of {u} is {v}",
            'phone_number': lambda u, v: f"📱 Phone number of {u} is {v}",
            'email': lambda u, v: f"📧 Email address of {u} is {v}",
            'address': lambda u, v: f"🏠 Home address of {u} is {v}",
            'dob': lambda u, v: f"🎂 Date of birth of {u} is {v}",
            'name': lambda u, v: f"👤 Name of {u} is {v}",
            'age': lambda u, v: f"📅 Age of {u} is {v}",
            'occupation': lambda u, v: f"💼 Occupation of {u} is {v}",
            'company': lambda u, v: f"🏢 Company of {u} is {v}",
            'relationship_status': lambda u, v: f"💑 Relationship status of {u} is {v}",
            'gender': lambda u, v: f"⚧ Gender of {u} is {v}",
            'nationality': lambda u, v: f"🌍 Nationality of {u} is {v}",
            'document_type': lambda u, v: f"📄 {u} shared a document of type {v}",
            'aadhaar_number': lambda u, v: f"🆔 Aadhaar number of {u} is {v}",
            'pan_number': lambda u, v: f"📇 PAN number of {u} is {v}",
            'family_member': lambda u, v: f"👨‍👩‍👧‍👦 Family member of {u}: {v}",
            'friend': lambda u, v: f"👫 Friend of {u}: {v}",
            'colleague': lambda u, v: f"🤝 Colleague of {u}: {v}",
            'contact_name': lambda u, v: f"📞 Contact of {u}: {v}",
            'relationship_type': lambda u, v: f"💕 {v} of {u}",
            'food_preference': lambda u, v: f"🍽️ Food preference of {u}: {v}",
            'restaurant_preference': lambda u, v: f"🏪 Preferred restaurant of {u}: {v}",
            'allergy': lambda u, v: f"⚠️ {u} is allergic to {v}",
            'service_provider': lambda u, v: f"🔧 Service provider of {u}: {v}",
            'vendor_name': lambda u, v: f"🛒 Vendor used by {u}: {v}",
            'habit': lambda u, v: f"🔄 Personal habit of {u}: {v}",
            'routine': lambda u, v: f"📋 Routine of {u}: {v}",
        }
        
        # Use specific formatter or fallback to generic
        formatter = fact_formatters.get(fact_type, lambda u, v: f"{fact_type.replace('_', ' ').title()} of {u} is {v}")
        return formatter(user_id, value)

    def merge_extracted_data(self, all_extracted: List[Dict]) -> Dict:
        """Merge extracted data from multiple chunks, removing duplicates"""
        merged = {"Layer1": [], "Layer2": [], "Layer3": [], "Layer4": []}
        
        for extracted in all_extracted:
            for layer in ["Layer1", "Layer2", "Layer3", "Layer4"]:
                if layer in extracted:
                    merged[layer].extend(extracted[layer])
        
        # Deduplicate based on fact type and value similarity
        for layer in merged:
            merged[layer] = self._deduplicate_facts(merged[layer])
        
        return merged

    def _deduplicate_facts(self, facts: List[Dict]) -> List[Dict]:
        """Remove duplicate facts based on type and value similarity"""
        if not facts:
            return facts
            
        deduped = []
        for fact in facts:
            is_duplicate = False
            fact_type = fact["detail"]["type"]
            fact_value = fact["detail"]["value"].lower().strip()
            
            for existing in deduped:
                existing_type = existing["detail"]["type"]
                existing_value = existing["detail"]["value"].lower().strip()
                
                # Same type and high similarity
                if (fact_type == existing_type and 
                    fuzz.ratio(fact_value, existing_value) > 85):
                    is_duplicate = True
                    # Keep the one with higher confidence
                    if fact["confidence"] > existing["confidence"]:
                        deduped.remove(existing)
                        deduped.append(fact)
                    break
            
            if not is_duplicate:
                deduped.append(fact)
        
        return deduped
        
    def call_llm_for_deduplication(self, prompt: str) -> Dict:
        """Call LLM specifically for deduplication tasks"""
        try:
            response = self.anthropic_client.messages.create(
                model=self.model_name,
                max_tokens=4000,
                temperature=0.1,  # Low temperature for consistency
                messages=[
                    {"role": "user", "content": prompt}
                ],
                system="""You are a memory system deduplication and validation expert. Your job is to return a valid JSON with deduplicated facts AND validate that all facts make sense.

DEDUPLICATION RULES:
- REMOVE ANY DUPLICATE VALUES THAT ARE PRESENT, OR ANYTHING THAT LOOKS REPEATED.
- IF THERE ARE 2 SAME KEYS, WITH DIFFERENT VALUES, MAKE SURE YOU ONLY ALLOW ONE INSTANCE WITH THE HIGHEST CONFIDENCE SCORE.
For example, if the same phone number/address is present multiple times, keep only one instance with the highest confidence.

VALIDATION & SENSE-CHECKING RULES:
- REMOVE any facts that don't make logical sense (e.g., "age": "blue", "phone": "happy", "name": "12345")
- REMOVE any facts with impossible values (e.g., age over 150, phone numbers with letters, invalid email formats)
- REMOVE any facts that are clearly nonsensical or corrupted data
- REMOVE any facts that contain only special characters, random symbols, or gibberish
- REMOVE any facts where the value doesn't match the fact type (e.g., an address in a phone number field)
- REMOVE any facts with extremely low confidence scores (below 0.3) that also seem questionable
- KEEP facts that make sense even if they seem unusual (e.g., uncommon names, foreign addresses)

FORMATTING RULES:
1. Return ONLY the JSON data with NO explanatory text, NO markdown formatting, and NO code blocks
2. Your entire response must be parseable as JSON
3. Maintain the exact same structure as the input JSON
4. CRITICAL: Preserve ALL evidence snippets exactly as given in the input. DO NOT change snippets to "No snippet available" or any placeholder text.
5. Never add new facts or modify existing valid data beyond merging duplicates and removing nonsensical entries
6. If duplicates have different confidence scores, keep the one with higher confidence
7. If duplicates have the same confidence, merge their evidence lists completely with no loss of information"""
            )
            
            # Extract the response content as text
            if not response or not response.content:
                print("⚠️ Empty response from LLM")
                return {}
                
            content = response.content[0].text
            print(f"Raw LLM response (first 100 chars): {content[:100]}...")
            
            # Try multiple methods to extract JSON
            
            # Method 1: Try to find JSON in code blocks
            json_match = re.search(r'```(?:json)?\s*([\s\S]*?)\s*```', content)
            if json_match:
                json_str = json_match.group(1)
                print("Found JSON in code block")
            else:
                # Method 2: Find first { and last }
                first_brace = content.find('{')
                last_brace = content.rfind('}')
                
                if first_brace != -1 and last_brace != -1 and first_brace < last_brace:
                    json_str = content[first_brace:last_brace+1]
                    print(f"Extracted JSON using brace matching: chars {first_brace} to {last_brace}")
                else:
                    # Method 3: Use the whole response
                    json_str = content
                    print("Using full response content for JSON parsing")
            
            # Clean up the JSON string
            json_str = re.sub(r'(?m)^//.*$', '', json_str)  # Remove comment lines
            json_str = json_str.strip()
            
            try:
                print(f"Attempting to parse JSON (length: {len(json_str)})")
                result = json.loads(json_str)
                print("✅ Successfully parsed JSON")
                return result
            except json.JSONDecodeError as e:
                print(f"⚠️ Failed to decode JSON from LLM response: {e}")
                print(f"JSON string start: {json_str[:200]}...")
                print(f"JSON string end: ...{json_str[-200:]}")
                
                # Fallback to raw extraction
                try:
                    # Attempt to rescue the response using regex
                    layer_pattern = r'"(Layer\d+)":\s*\[([\s\S]*?)\]'
                    layers = re.findall(layer_pattern, json_str)
                    
                    if layers:
                        print("🛠️ Attempting fallback regex extraction")
                        result = {}
                        for layer_name, layer_content in layers:
                            result[layer_name] = []
                        return result
                except Exception as regex_error:
                    print(f"Regex fallback failed: {regex_error}")
                    
                return {}
                
        except Exception as e:
            print(f"⚠️ Error calling LLM for deduplication: {e}")
            return {}

    async def deduplicate_with_llm(self, extracted_data: Dict, user_id: str) -> Dict:
        """Use LLM to deduplicate facts and merge evidence when appropriate
        
        This takes the raw extraction results and has the LLM analyze them to:
        1. Remove exact duplicates, keeping the highest confidence version
        2. Merge duplicates with same confidence by combining their evidence
        3. Maintain the 4-layer memory structure
        """
        if not extracted_data:
            print(f"No data to deduplicate for user {user_id}")
            return {}
            
        # Convert extracted data to a format suitable for LLM processing
        facts_by_layer = {}
        for layer, nodes in extracted_data.items():
            facts_by_layer[layer] = []
            for node in nodes:
                # Create a simplified representation for LLM
                evidence_snippets = []
                for ev in node['evidence']:
                    evidence_snippets.append({
                        'message_id': ev.get('message_id', 'unknown'),
                        'snippet': ev.get('message_snippet', ev.get('snippet', ''))  # Handle both formats
                    })
                    
                fact = {
                    'type': node['detail']['type'],
                    'value': node['detail']['value'],
                    'confidence': node['confidence'],
                    'evidence': evidence_snippets
                }
                facts_by_layer[layer].append(fact)
        
        # Create prompt for LLM deduplication and validation
        prompt = f"""
Review, validate, and deduplicate the following memory facts for user {user_id}.

DEDUPLICATION INSTRUCTIONS:
1. PHONE NUMBER DEDUPLICATION: If there are multiple phone numbers for the same person, keep ONLY the one with the highest confidence score. Different formats of the same number (+91 prefix vs without) should be considered duplicates.
2. ADDRESS DEDUPLICATION: If there are multiple addresses for the same person, keep ONLY the one with the highest confidence score. Similar addresses should be considered duplicates.
3. EMAIL DEDUPLICATION: If there are multiple emails, keep all distinct email addresses as people often have multiple emails.
4. RELATIONSHIP DEDUPLICATION: Remove any relationship entries that are just generic terms like "wife", "husband", "nephew" without specific names. Only keep relationships with actual names.
5. GENERAL RULE: For all other fact types, remove duplicates keeping only the highest confidence version for each unique fact.

VALIDATION & SENSE-CHECKING INSTRUCTIONS:
6. DATA TYPE VALIDATION: Remove facts where the value doesn't match the expected data type:
   - Names should be text, not numbers or symbols
   - Phone numbers should contain only digits, spaces, +, -, (, )
   - Ages should be reasonable numbers (0-150)
   - Emails should have valid email format
   - Addresses should be coherent location text
7. LOGICAL SENSE CHECK: Remove facts that are clearly nonsensical:
   - Random character combinations (e.g., "asdfgh", "123!@#")
   - Values that are clearly system errors or corrupted data
   - Facts with impossible combinations (e.g., age: "red", name: "999999")
8. CONFIDENCE FILTERING: Remove facts with very low confidence (<0.3) that also seem questionable or corrupted
9. PRESERVE UNUSUAL BUT VALID: Keep facts that seem unusual but are logically valid (foreign names, uncommon addresses, etc.)

STRUCTURAL RULES:
10. If duplicates have identical confidence scores, merge their evidence lists into one comprehensive fact.
11. Maintain the 4-layer structure (Layer1: Basic Personal Info, Layer2: Documents, Layer3: Relationships, Layer4: Preferences).
12. Do not add, remove, or modify any actual data values that are valid - simply reorganize, deduplicate, and remove nonsensical entries.
13. CRITICAL: Preserve ALL evidence snippets exactly as given in the input. DO NOT replace snippets with "No snippet available" or any placeholder text.
14. Return ONLY a valid JSON with no additional text or explanation.

IMPORTANT: Your response must be ONLY the cleaned and deduplicated JSON data with nothing else. Do not include any explanations, markdown formatting, or text outside the JSON. Return the JSON directly with all evidence snippets preserved exactly as provided.

When processing evidence, maintain the exact field names as provided in input (message_id and either 'snippet' or 'message_snippet').

RELATIONSHIP FILTERING RULES:
- REMOVE: "relationship": "wife" (no name specified)
- REMOVE: "family_member": "nephew" (no name specified)  
- KEEP: "spouse": "Sarah" (specific name provided)
- KEEP: "family_member": "Mike Johnson" (specific name provided)

PHONE NUMBER DEDUPLICATION EXAMPLE:
These are 3 different phone numbers for the same person. Keep ONLY the highest confidence one: "9870781578" (confidence: 0.95)

Input facts: {json.dumps(facts_by_layer, indent=2)}
"""

        try:
            # Call LLM for deduplication
            deduplicated_data = self.call_llm_for_deduplication(prompt)
            
            # Verify we got a valid response
            if not deduplicated_data:
                print("⚠️ No valid data from LLM deduplication. Applying fallback deduplication.")
                return self._apply_fallback_deduplication(extracted_data, user_id)
                
            # Verify that we have all the expected layers
            missing_layers = set(extracted_data.keys()) - set(deduplicated_data.keys())
            if missing_layers:
                print(f"⚠️ LLM response is missing layers: {missing_layers}. Applying fallback deduplication.")
                return self._apply_fallback_deduplication(extracted_data, user_id)
            
            print(f"\n🔍 LLM deduplication complete for {user_id}")
            
            # Count before/after
            before_count = sum(len(nodes) for nodes in extracted_data.values())
            after_count = sum(len(nodes) for nodes in deduplicated_data.values())
            
            # Safety check - if LLM removed all facts, something went wrong
            if after_count == 0 and before_count > 0:
                print("⚠️ LLM deduplication removed ALL facts! Applying fallback deduplication.")
                return self._apply_fallback_deduplication(extracted_data, user_id)
                
            print(f"   Facts before: {before_count}, after: {after_count}, removed: {before_count - after_count}")
            
            # Convert back to the original format structure
            result = {}
            for layer, facts in deduplicated_data.items():
                result[layer] = []
                for fact in facts:
                    # Handle potential missing keys with defaults
                    if 'type' not in fact or 'value' not in fact:
                        print(f"⚠️ Skipping malformed fact (missing type/value): {fact}")
                        continue
                        
                    # Reconstruct evidence in the expected format - strictly preserve original snippets
                    evidence = []
                    if 'evidence' in fact and isinstance(fact['evidence'], list):
                        for ev in fact['evidence']:
                            message_id = ev.get('message_id', 'unknown')
                            # Handle both 'snippet' and 'message_snippet' from LLM responses
                            snippet = ev.get('snippet', ev.get('message_snippet', ''))
                            
                            # Ensure snippet is preserved exactly as in the original
                            if not snippet or snippet == "No snippet available":
                                print(f"⚠️ Warning: Missing or placeholder snippet for message {message_id}")
                                
                            evidence.append({
                                'message_id': message_id,
                                'snippet': snippet
                            })
                    else:
                        print(f"⚠️ Warning: Missing evidence list in fact: {fact['type']}: {fact['value']}")
                        # Create minimal evidence to avoid UI errors
                        evidence = [{'message_id': 'unknown', 'snippet': 'Evidence missing during deduplication'}]
                    
                    # Reconstruct node in the expected format
                    node = {
                        'detail': {
                            'type': fact['type'],
                            'value': fact['value']
                        },
                        'confidence': fact.get('confidence', 0.8),  # Default if missing
                        'evidence': evidence
                    }
                    result[layer].append(node)
            
            return result
        except Exception as e:
            print(f"⚠️ Error during LLM deduplication: {e}")
            print("Applying fallback deduplication to ensure duplicates are removed")
            return self._apply_fallback_deduplication(extracted_data, user_id)

    def _apply_fallback_deduplication(self, extracted_data: Dict, user_id: str) -> Dict:
        """Apply rule-based deduplication when LLM deduplication fails
        
        This ensures that deduplication ALWAYS happens, even if LLM fails.
        """
        print(f"\n🔧 Applying fallback deduplication for {user_id}")
        
        result = {}
        total_removed = 0
        
        for layer, nodes in extracted_data.items():
            if not nodes:
                result[layer] = []
                continue
                
            # Apply rule-based deduplication with special handling for phone numbers
            deduplicated_nodes = self._deduplicate_facts_enhanced(nodes, layer)
            result[layer] = deduplicated_nodes
            
            removed_count = len(nodes) - len(deduplicated_nodes)
            total_removed += removed_count
            
            if removed_count > 0:
                print(f"   {layer}: {len(nodes)} → {len(deduplicated_nodes)} facts (removed {removed_count} duplicates)")
        
        before_count = sum(len(nodes) for nodes in extracted_data.values())
        after_count = sum(len(nodes) for nodes in result.values())
        
        print(f"   Facts before: {before_count}, after: {after_count}, removed: {total_removed}")
        print(f"✅ Fallback deduplication complete for {user_id}")
        
        return result
    
    def _deduplicate_facts_enhanced(self, facts: List[Dict], layer: str) -> List[Dict]:
        """Enhanced rule-based deduplication with special handling for different fact types"""
        if not facts:
            return []
        
        deduped = []
        
        # Group facts by type for specialized deduplication
        facts_by_type = {}
        for fact in facts:
            fact_type = fact['detail']['type']
            if fact_type not in facts_by_type:
                facts_by_type[fact_type] = []
            facts_by_type[fact_type].append(fact)
        
        # Apply type-specific deduplication rules
        for fact_type, type_facts in facts_by_type.items():
            if fact_type == 'phone_number':
                # For phone numbers: keep only the highest confidence one
                deduplicated = self._deduplicate_phone_numbers(type_facts)
            elif fact_type in ['address', 'email']:
                # For addresses and emails: more sophisticated deduplication
                deduplicated = self._deduplicate_contact_info(type_facts, fact_type)
            else:
                # For other types: use standard fuzzy matching
                deduplicated = self._deduplicate_facts(type_facts)
            
            deduped.extend(deduplicated)
        
        return deduped
    
    def _deduplicate_phone_numbers(self, phone_facts: List[Dict]) -> List[Dict]:
        """Keep only the highest confidence phone number"""
        if not phone_facts:
            return []
        
        # Sort by confidence (highest first)
        sorted_phones = sorted(phone_facts, key=lambda x: x['confidence'], reverse=True)
        
        # Keep only the highest confidence phone number
        best_phone = sorted_phones[0]
        
        print(f"      Phone deduplication: {len(phone_facts)} → 1 (kept highest confidence: {best_phone['detail']['value']})")
        
        return [best_phone]
    
    def _deduplicate_contact_info(self, contact_facts: List[Dict], fact_type: str) -> List[Dict]:
        """Deduplicate addresses/emails with fuzzy matching"""
        if not contact_facts:
            return []
        
        deduped = []
        
        for fact in contact_facts:
            fact_value = fact['detail']['value'].lower().strip()
            is_duplicate = False
            
            for existing in deduped:
                existing_value = existing['detail']['value'].lower().strip()
                
                # For emails: exact match after normalization
                if fact_type == 'email':
                    if fact_value == existing_value:
                        is_duplicate = True
                        break
                
                # For addresses: fuzzy matching
                elif fact_type == 'address':
                    if (fact_value == existing_value or 
                        fuzz.ratio(fact_value, existing_value) > 85):
                        is_duplicate = True
                        # Keep the one with higher confidence
                        if fact["confidence"] > existing["confidence"]:
                            deduped.remove(existing)
                            deduped.append(fact)
                        break
            
            if not is_duplicate:
                deduped.append(fact)
        
        if len(contact_facts) != len(deduped):
            print(f"      {fact_type} deduplication: {len(contact_facts)} → {len(deduped)}")
        
        return deduped

    async def process_json(self, input_json: List[Dict], user_id: str, force_reprocess: bool = False) -> Dict:
        """Process the JSON: chunk messages, call LLM for each chunk, merge and store results"""
        
        # Check if file has already been processed (unless force reprocess)
        if not force_reprocess and self.db_enabled:
            already_processed = await supabase_manager.is_file_processed(user_id)
            if already_processed:
                print(f"⏭️  {user_id} already processed. Skipping. Use force_reprocess=True to reprocess.")
                return {"message": "Already processed", "skipped": True}
        
        # Flatten messages, preserving order and adding metadata
        all_messages = []
        for conv_idx, conv in enumerate(input_json):
            for query in conv.get("user_queries", []):
                query = query.copy()
                query["sender"] = "User"
                query["conversation_id"] = conv_idx
                # Preserve original message_id, use as id for processing
                query["id"] = query.get("message_id", f"unknown_user_{conv_idx}")
                all_messages.append(query)
            for reply in conv.get("team_replies", []):
                reply = reply.copy()
                reply["sender"] = "Team"
                reply["conversation_id"] = conv_idx
                # Preserve original message_id, use as id for processing
                reply["id"] = reply.get("message_id", f"unknown_team_{conv_idx}")
                all_messages.append(reply)

        print(f"Processing {len(all_messages)} messages for user {user_id}")
        
        # Create chunks for processing
        chunks = self.chunk_messages(all_messages)
        print(f"Split into {len(chunks)} chunks (chunk_size={self.chunk_size}, overlap={self.overlap_size})")
        
        # Process each chunk
        all_extracted_data = []
        for i, chunk in enumerate(chunks):
            print(f"\nProcessing chunk {i+1}/{len(chunks)} ({len(chunk)} messages)")
            
            prompt = self.prepare_comprehensive_llm_prompt(chunk, user_id)
            extracted_data = self.call_llm_for_extraction(prompt)
            
            if extracted_data:
                # Count nodes in this chunk
                total_nodes = sum(len(nodes) for nodes in extracted_data.values() if isinstance(nodes, list))
                print(f"  Extracted {total_nodes} nodes from chunk {i+1}")
                all_extracted_data.append(extracted_data)
            else:
                print(f"  No data extracted from chunk {i+1}")
        
        # Merge all extracted data and remove duplicates
        if all_extracted_data:
            merged_data = self.merge_extracted_data(all_extracted_data)
            
            # Print summary
            total_nodes = sum(len(nodes) for nodes in merged_data.values())
            print(f"\nMerged Results for user {user_id}: {total_nodes} total nodes")
            for layer, nodes in merged_data.items():
                if nodes:
                    print(f"  {layer}: {len(nodes)} nodes")
            
            # Apply LLM deduplication to get a refined version with no duplicates
            print("\n🧹 Applying LLM deduplication to remove duplicates and merge evidence...")
            try:
                deduplicated_data = await self.deduplicate_with_llm(merged_data, user_id)
                print("✅ LLM deduplication completed successfully")
            except Exception as dedup_error:
                print(f"❌ Error during LLM deduplication: {str(dedup_error)}")
                raise dedup_error
            
            # Print deduplication summary
            dedup_total_nodes = sum(len(nodes) for nodes in deduplicated_data.values())
            print(f"\nDeduplicated Results for user {user_id}: {dedup_total_nodes} total nodes")
            for layer, nodes in deduplicated_data.items():
                if nodes:
                    print(f"  {layer}: {len(nodes)} nodes")
            
            # Store deduplicated data in database
            print(f"\n💾 Storing {dedup_total_nodes} deduplicated nodes in database...")
            try:
                storage_result = await self.store_in_db(deduplicated_data, user_id)
                print("✅ Database storage completed successfully")
            except Exception as storage_error:
                print(f"❌ Error during database storage: {str(storage_error)}")
                raise storage_error
            
            # Mark file as processed
            if self.db_enabled:
                print(f"\n📝 Marking file as processed for user {user_id}...")
                try:
                    await supabase_manager.mark_file_processed(user_id, dedup_total_nodes)
                    print("✅ File marked as processed successfully")
                except Exception as mark_error:
                    print(f"❌ Error marking file as processed: {str(mark_error)}")
                    raise mark_error
            
            # Return only the newly stored facts from this iteration
            newly_stored_facts = storage_result.get("newly_stored_nodes", {})
            print(f"\n🎯 Returning {sum(len(nodes) for nodes in newly_stored_facts.values())} newly stored facts for this iteration")
            
            return newly_stored_facts
        else:
            print(f"No data extracted for user {user_id}")
            return {}

    async def reprocess_rejected_fact(self, user_id: str, fact_type: str, layer: str, 
                               excluded_message_ids: List[str], original_node_id: str) -> Dict:
        """Reprocess a specific rejected fact type by re-analyzing with focused prompt"""
        
        # Load the user's JSON file
        input_path = os.path.join("input_jsons", f"{user_id}.json")
        if not os.path.exists(input_path):
            print(f"Warning: {input_path} not found for reprocessing.")
            return {}
        
        with open(input_path, 'r', encoding='utf-8') as f:
            input_data = json.load(f)
        
        # Flatten messages, preserving order
        all_messages = []
        for conv in input_data:
            for query in conv.get("user_queries", []):
                query = query.copy()
                query["sender"] = "User"
                # Preserve original message_id, use as id for processing
                query["id"] = query.get("message_id", "unknown_user")
                all_messages.append(query)
            for reply in conv.get("team_replies", []):
                reply = reply.copy()
                reply["sender"] = "Team"
                # Preserve original message_id, use as id for processing
                reply["id"] = reply.get("message_id", "unknown_team")
                all_messages.append(reply)
        
        # Filter out messages that were used in the original (rejected) extraction
        filtered_messages = [
            msg for msg in all_messages 
            if msg.get("id") not in excluded_message_ids
        ]
        
        print(f"Reprocessing {fact_type} for {user_id}")
        print(f"Original context had {len(excluded_message_ids)} messages, searching in {len(filtered_messages)} remaining messages")
        
        # Create chunks from filtered messages
        chunks = self.chunk_messages(filtered_messages)
        
        layer_number = layer.split("_")[-1] if "_" in layer else layer.replace("layer", "").replace("Layer", "")
        layer_key = f"Layer{layer_number}"
        extracted_data = {layer_key: []}
        
        print(f"Processing {len(chunks)} chunks for reprocessing")
        
        for i, chunk in enumerate(chunks):
            print(f"  Reprocessing chunk {i+1}: {len(chunk)} messages")
            
            # Create a specialized prompt for reprocessing
            prompt = self.prepare_reprocessing_prompt(chunk, layer_key, fact_type)
            llm_output = self.call_llm_for_extraction(prompt)
            nodes = llm_output.get(layer_key, [])
            
            if nodes:
                print(f"    Extracted {len(nodes)} nodes on reprocessing")
                extracted_data[layer_key].extend(nodes)
            else:
                print(f"    No nodes extracted on reprocessing")
        
        # Deduplicate the reprocessed results
        if extracted_data[layer_key]:
            extracted_data[layer_key] = self._deduplicate_facts(extracted_data[layer_key])
            await self.store_reprocessed_nodes(extracted_data, user_id, original_node_id)
        else:
            print(f"Reprocessing complete: No {fact_type} found in alternative contexts")
        
        return extracted_data

    def prepare_reprocessing_prompt(self, chunk: List[Dict[str, str]], layer: str, fact_type: str) -> str:
        """Prepare specialized prompt for reprocessing rejected facts"""
        deduped_chunk = self.deduplicate_messages(chunk)
        context_text = "\n".join([
            f"[{msg.get('id', i)}] {msg.get('sender', 'Unknown')}: {msg.get('message', '')}" 
            for i, msg in enumerate(deduped_chunk)
        ])
        layer_number = layer.replace("Layer", "")
        
        layer_descriptions = {
            "1": "Basic Personal Information (name, age, address, phone, email, DOB, nationality, gender, blood group, relationship status)",
            "2": "Information from Documents Shared (Aadhaar card, PAN card, driving license, voter ID, birth certificate, insurance, rent agreement, utility bills)",
            "3": "Loved Ones & Relations (family members, friends, colleagues, roommates, partners with their names, relationships, and contact details)",
            "4": "Preferences, Vendors, Standing Instructions (food preferences, favorite restaurants, service providers, vendors, habits, routines, standing orders)"
        }
        
        prompt = f"""
You are reprocessing a REJECTED extraction for a personal assistant memory system. A previous extraction of "{fact_type}" was rejected by reviewers.

LAYER {layer_number}: {layer_descriptions[layer_number]}

SPECIFIC TASK: Find "{fact_type}" information in this conversation chunk with extra care and precision.

CONVERSATION CHUNK:
{context_text}

INSTRUCTIONS:
1. Focus ONLY on finding clear, unambiguous "{fact_type}" information
2. This is a REPROCESSING attempt - be more careful and precise than usual
3. Look for casual conversational patterns, not just formal statements
4. Only extract if you find very clear evidence with high confidence (0.8+)
5. If not found clearly, return empty results
6. Reference specific message IDs as evidence

RESPONSE FORMAT (JSON only):
{{
  "{layer}": [
    {{
      "detail": {{"type": "{fact_type}", "value": "extracted_value"}},
      "confidence": 0.85,
      "evidence": [
        {{"message_id": "id", "message_snippet": "relevant_part_of_message"}}
      ],
      "timestamp": "2025-09-13 00:00:00"
    }}
  ]
}}

Only return JSON. If no clear "{fact_type}" information is found, return empty array.
"""
        return prompt

    async def store_reprocessed_nodes(self, extracted_nodes: Dict, user_id: str, original_node_id: str):
        """Store reprocessed nodes with link to original rejected node"""
        if not self.db_enabled:
            print(f"\n=== REPROCESSED DATA FOR USER: {user_id} (DB DISABLED) ===")
            for layer, nodes in extracted_nodes.items():
                print(f"\n{layer}:")
                for i, node in enumerate(nodes, 1):
                    print(f"  {i}. Type: {node['detail']['type']}")
                    print(f"     Value: {node['detail']['value']}")
                    print(f"     Confidence: {node['confidence']}")
                    print(f"     Evidence: {len(node['evidence'])} message(s)")
                    print(f"     Original Node ID: {original_node_id}")
            print("=" * 50)
            return
        
        try:
            stored_nodes = 0
            
            for layer, nodes in extracted_nodes.items():
                layer_number = int(layer.replace("Layer", ""))
                
                for node in nodes:
                    detail = node["detail"]
                    confidence = node["confidence"]
                    evidence = node["evidence"]
                    
                    # Safely extract required fields  
                    try:
                        fact_type = detail.get("type", "unknown")
                        fact_value = detail.get("value", "N/A")
                        
                        if fact_value == "N/A":
                            print(f"⚠️  Warning: Missing 'value' field in detail: {detail}")
                    
                        # Format concluded fact for human readability
                        concluded_fact = self._format_concluded_fact(user_id, fact_type, fact_value)
                        
                        # Store reprocessed memory node with link to original
                        node_id = await supabase_manager.store_memory_node(
                            user_phone=user_id,
                            layer=layer_number,
                            fact_type=fact_type,
                            content=fact_value,
                            concluded_fact=concluded_fact,
                            confidence=confidence,
                            evidence=evidence,
                            extraction_method='reprocess',
                            parent_update_id=original_node_id
                        )
                        
                        if node_id:
                            stored_nodes += 1
                            print(f"  ✅ Stored reprocessed {fact_type}: {fact_value} (Node ID: {node_id})")
                        else:
                            print(f"  ⚠️  Failed to store reprocessed {fact_type}: {fact_value}")
                            
                    except Exception as detail_error:
                        print(f"❌ Error processing detail: {str(detail_error)}")
                        print(f"❌ Detail structure: {detail}")
                        continue  # Skip this detail and continue with next one
            
            # Mark original node as reprocessed
            if stored_nodes > 0:
                await supabase_manager.mark_reprocessing_complete(original_node_id)
                print(f"\n📊 Reprocessing Summary for {user_id}:")
                print(f"   Stored: {stored_nodes} reprocessed nodes")
                print(f"   Linked to original node: {original_node_id}")
            
        except Exception as e:
            print(f"❌ Database error during reprocessing: {e}")

    def main(self):
        """Standalone batch processing for all JSON files"""
        input_folder = "input_jsons"
        input_files = [
            "RahulSingh.json",
            "MonarkMoolchandani.json",
            "ManuJain.json",
            "Gaurav-Sherlocksai.json",
            "Anurag.json",
            "AdityaShetty.json"
        ]

        for filename in input_files:
            input_path = os.path.join(input_folder, filename)
            if not os.path.exists(input_path):
                print(f"Warning: {input_path} not found, skipping.")
                continue

            print(f"\n{'='*50}")
            print(f"Processing {input_path}")
            print(f"{'='*50}")
            
            with open(input_path, 'r', encoding='utf-8') as f:
                input_data = json.load(f)

            user_id = filename.split(".")[0]
            
            import asyncio
            asyncio.run(self.process_json(input_data, user_id))
            print(f"Finished processing {input_path}")

if __name__ == "__main__":
    extractor = JSONContextExtractor(chunk_size=100, overlap_size=20)  # Configurable chunk sizes
    extractor.main()