Update pages/linkedin_extractor.py

pages/linkedin_extractor.py

@@ -20,36 +20,56 @@ st.set_page_config(
 )
 
 def get_embeddings():
-    """Initialize embeddings with multiple fallback options"""
+    """Initialize embeddings with better fallback options"""
     try:
-        # Try multiple embedding models
+        # Try multiple embedding models with different approaches
         model_options = [
-            "sentence-transformers/all-MiniLM-L6-v2",  # Default
-            "sentence-transformers/paraphrase-albert-small-v2",  # Smaller alternative
-            "sentence-transformers/all-mpnet-base-v2"  # Higher quality
+            "sentence-transformers/all-MiniLM-L6-v2",
+            "sentence-transformers/all-mpnet-base-v2", 
+            "BAAI/bge-small-en-v1.5",
+            "sentence-transformers/paraphrase-MiniLM-L6-v2"
         ]
         
         for model_name in model_options:
             try:
+                st.info(f"🔄 Trying to load: {model_name}")
                 embeddings = HuggingFaceEmbeddings(
                     model_name=model_name,
                     model_kwargs={'device': 'cpu'},
-                    encode_kwargs={'normalize_embeddings': True}
+                    encode_kwargs={
+                        'normalize_embeddings': True,
+                        'batch_size': 32
+                    }
                 )
-                st.success(f"✅ Loaded embeddings: {model_name.split('/')[-1]}")
-                return embeddings
+                # Test the embeddings
+                test_text = "Hello world"
+                test_embedding = embeddings.embed_query(test_text)
+                if test_embedding and len(test_embedding) > 0:
+                    st.success(f"✅ Loaded embeddings: {model_name.split('/')[-1]}")
+                    return embeddings
             except Exception as e:
+                st.warning(f"⚠️ Failed to load {model_name}: {str(e)}")
                 continue
-                
-        st.error("❌ All embedding models failed to load")
-        return None
         
+        # If all models fail, try a simpler approach
+        st.warning("🔄 Trying fallback embedding method...")
+        try:
+            embeddings = HuggingFaceEmbeddings(
+                model_name="sentence-transformers/all-MiniLM-L6-v2",
+                cache_folder="/tmp/embeddings"
+            )
+            st.success("✅ Loaded fallback embeddings")
+            return embeddings
+        except Exception as e:
+            st.error(f"❌ Fallback also failed: {e}")
+            return None
+                
     except Exception as e:
         st.error(f"❌ Embeddings error: {e}")
         return None
 
 def get_llm():
-    """Initialize Mistral 7B LLM - Best for analysis"""
+    """Initialize Mistral 7B LLM with better error handling"""
     try:
         api_key = os.getenv('HUGGINGFACEHUB_API_TOKEN')
         if not api_key:
@@ -65,24 +85,79 @@ def get_llm():
             """)
             return None
         
-        # Using Mistral 7B - Best balance of quality and accessibility
-        llm = HuggingFaceHub(
-            repo_id="mistralai/Mistral-7B-Instruct-v0.1",
-            huggingfacehub_api_token=api_key,
-            model_kwargs={
-                "temperature": 0.7,
-                "max_length": 2048,
-                "max_new_tokens": 512,
-                "top_p": 0.95,
-                "repetition_penalty": 1.1,
-                "do_sample": True
-            }
-        )
-        return llm
+        # Try multiple models
+        model_options = [
+            "mistralai/Mistral-7B-Instruct-v0.1",
+            "HuggingFaceH4/zephyr-7b-beta",
+            "google/flan-t5-large"
+        ]
+        
+        for model_id in model_options:
+            try:
+                st.info(f"🔄 Trying to load: {model_id}")
+                llm = HuggingFaceHub(
+                    repo_id=model_id,
+                    huggingfacehub_api_token=api_key,
+                    model_kwargs={
+                        "temperature": 0.7,
+                        "max_length": 2048,
+                        "max_new_tokens": 512,
+                        "top_p": 0.95,
+                        "repetition_penalty": 1.1,
+                        "do_sample": True
+                    }
+                )
+                # Test the model
+                test_response = llm.invoke("Hello")
+                if test_response:
+                    st.success(f"✅ Loaded model: {model_id.split('/')[-1]}")
+                    return llm
+            except Exception as e:
+                st.warning(f"⚠️ Failed to load {model_id}: {str(e)}")
+                continue
+        
+        st.error("❌ All AI models failed to load")
+        return None
+        
     except Exception as e:
         st.error(f"❌ AI Model error: {e}")
         return None
 
+def simple_chat_analysis(user_input, extracted_data):
+    """Simple chat analysis without embeddings as fallback"""
+    try:
+        if not extracted_data:
+            return "No data available for analysis."
+        
+        content_blocks = extracted_data.get('content_blocks', [])
+        page_info = extracted_data.get('page_info', {})
+        
+        # Create context from extracted data
+        context = f"Page Title: {page_info.get('title', 'N/A')}\n"
+        context += f"Content Type: {extracted_data.get('data_type', 'N/A')}\n"
+        context += f"Extracted Content:\n"
+        
+        for i, block in enumerate(content_blocks[:5]):  # Limit context
+            context += f"Block {i+1}: {block}\n"
+        
+        # Simple rule-based responses
+        user_input_lower = user_input.lower()
+        
+        if any(word in user_input_lower for word in ['summary', 'summarize', 'overview']):
+            return f"Based on the LinkedIn data, here's a summary:\n\nTitle: {page_info.get('title', 'N/A')}\nContent Type: {extracted_data.get('data_type', 'N/A')}\nTotal Content Blocks: {len(content_blocks)}\nKey Content: {content_blocks[0][:200] if content_blocks else 'No content available'}..."
+        
+        elif any(word in user_input_lower for word in ['skills', 'expertise', 'technologies']):
+            return "I can analyze the content for skills and expertise. The extracted data shows professional information that can be reviewed for specific skills mentioned in the content blocks."
+        
+        elif any(word in user_input_lower for word in ['experience', 'background', 'career']):
+            return "The LinkedIn data contains professional experience information. I can help you analyze the career background and work history mentioned in the profile."
+        
+        else:
+            return f"I've analyzed the LinkedIn data. {page_info.get('title', 'The profile')} contains {len(content_blocks)} content blocks with professional information. You can ask me about summaries, skills, experience, or specific details from the extracted content."
+
+    except Exception as e:
+        return f"Analysis error: {str(e)}"
+
 def extract_linkedin_data(url, data_type):
     """Extract data from LinkedIn URLs"""
     try:
@@ -154,64 +229,75 @@ def extract_linkedin_data(url, data_type):
         return {"error": f"Extraction error: {str(e)}", "status": "error"}
 
 def process_extracted_data(extracted_data):
-    """Process extracted data for AI analysis"""
+    """Process extracted data for AI analysis with fallbacks"""
     if not extracted_data or extracted_data.get("status") != "success":
         return None, []
     
-    page_info = extracted_data['page_info']
-    content_blocks = extracted_data['content_blocks']
-    
-    # Structure the data for AI
-    all_text = f"LINKEDIN DATA ANALYSIS REPORT\n"
-    all_text += "=" * 70 + "\n\n"
-    all_text += f"📄 PAGE INFORMATION:\n"
-    all_text += f"Title: {page_info['title']}\n"
-    all_text += f"URL: {page_info['url']}\n"
-    all_text += f"Type: {extracted_data['data_type'].upper()}\n"
-    all_text += f"Extracted: {extracted_data['extraction_time']}\n"
-    all_text += f"Response Code: {page_info['response_code']}\n"
-    all_text += f"Content Length: {page_info['content_length']} characters\n\n"
-    
-    all_text += f"📊 CONTENT ANALYSIS:\n"
-    all_text += f"Total Content Blocks: {len(content_blocks)}\n\n"
-    
-    # Add content blocks
-    for i, block in enumerate(content_blocks[:20]):
-        all_text += f"--- CONTENT BLOCK {i+1} ---\n"
-        all_text += f"Words: {len(block.split())} | Characters: {len(block)}\n"
-        all_text += f"Content: {block}\n\n"
-    
-    all_text += "=" * 70 + "\n"
-    all_text += "END OF EXTRACTION REPORT"
-    
-    # Split into chunks
-    splitter = CharacterTextSplitter(
-        separator="\n",
-        chunk_size=1000,
-        chunk_overlap=200,
-        length_function=len
-    )
-    
-    chunks = splitter.split_text(all_text)
-    documents = [Document(page_content=chunk) for chunk in chunks]
-    
-    # Create vector store
     try:
+        page_info = extracted_data['page_info']
+        content_blocks = extracted_data['content_blocks']
+        
+        # Structure the data for AI
+        all_text = f"LINKEDIN DATA ANALYSIS REPORT\n"
+        all_text += "=" * 70 + "\n\n"
+        all_text += f"📄 PAGE INFORMATION:\n"
+        all_text += f"Title: {page_info['title']}\n"
+        all_text += f"URL: {page_info['url']}\n"
+        all_text += f"Type: {extracted_data['data_type'].upper()}\n"
+        all_text += f"Extracted: {extracted_data['extraction_time']}\n"
+        all_text += f"Response Code: {page_info['response_code']}\n"
+        all_text += f"Content Length: {page_info['content_length']} characters\n\n"
+        
+        all_text += f"📊 CONTENT ANALYSIS:\n"
+        all_text += f"Total Content Blocks: {len(content_blocks)}\n\n"
+        
+        # Add content blocks
+        for i, block in enumerate(content_blocks[:10]):  # Limit for performance
+            all_text += f"--- CONTENT BLOCK {i+1} ---\n"
+            all_text += f"Words: {len(block.split())} | Characters: {len(block)}\n"
+            all_text += f"Content: {block}\n\n"
+        
+        all_text += "=" * 70 + "\n"
+        all_text += "END OF EXTRACTION REPORT"
+        
+        # Try to create vector store
         embeddings = get_embeddings()
         if embeddings is None:
-            return None, []
+            st.warning("⚠️ Using simple text processing (embeddings unavailable)")
+            # Return simple document structure
+            documents = [Document(page_content=all_text)]
+            return "simple", documents
+        
+        # Split into chunks
+        splitter = CharacterTextSplitter(
+            separator="\n",
+            chunk_size=800,  # Smaller for better performance
+            chunk_overlap=100,
+            length_function=len
+        )
+        
+        chunks = splitter.split_text(all_text)
+        documents = [Document(page_content=chunk) for chunk in chunks]
+        
+        # Create vector store
         vectorstore = FAISS.from_documents(documents, embeddings)
         return vectorstore, chunks
+        
     except Exception as e:
-        st.error(f"Vector store creation failed: {e}")
+        st.error(f"❌ Processing failed: {e}")
+        # Fallback: return simple structure
+        if extracted_data:
+            simple_doc = Document(page_content=f"LinkedIn Data: {extracted_data['page_info']['title']}")
+            return "simple", [simple_doc]
         return None, []
 
 def create_chatbot(vectorstore):
-    """Create conversational chatbot with Mistral"""
+    """Create conversational chatbot with fallbacks"""
     try:
         llm = get_llm()
         if llm is None:
-            return None
+            st.warning("⚠️ Using simple chat analysis (AI model unavailable)")
+            return "simple"
         
         memory = ConversationBufferMemory(
             memory_key="chat_history",
@@ -221,22 +307,20 @@ def create_chatbot(vectorstore):
         
         chain = ConversationalRetrievalChain.from_llm(
             llm=llm,
-            retriever=vectorstore.as_retriever(search_kwargs={"k": 4}),
+            retriever=vectorstore.as_retriever(search_kwargs={"k": 3}),
             memory=memory,
             return_source_documents=True,
             output_key="answer"
         )
         return chain
     except Exception as e:
-        st.error(f"Failed to create chatbot: {str(e)}")
-        return None
+        st.error(f"❌ Chatbot creation failed: {str(e)}")
+        return "simple"
 
 def clear_chat_history():
     """Clear chat history while keeping extracted data"""
-    if "vectorstore" in st.session_state and st.session_state.vectorstore:
-        st.session_state.chatbot = create_chatbot(st.session_state.vectorstore)
-        st.session_state.chat_history = []
-        st.success("🔄 Chat history cleared! Starting fresh conversation.")
+    st.session_state.chat_history = []
+    st.success("🔄 Chat history cleared! Starting fresh conversation.")
 
 def display_metrics(extracted_data):
     """Display extraction metrics"""
@@ -336,16 +420,24 @@ def main():
                         st.session_state.extracted_data = extracted_data
                         st.session_state.current_url = url_to_use
                         
-                        # Process for AI
-                        vectorstore, chunks = process_extracted_data(extracted_data)
-                        if vectorstore:
+                        # Process for AI (with fallbacks)
+                        result = process_extracted_data(extracted_data)
+                        if result:
+                            vectorstore, chunks = result
                             st.session_state.vectorstore = vectorstore
-                            st.session_state.chatbot = create_chatbot(vectorstore)
+                            
+                            # Create chatbot (with fallbacks)
+                            chatbot = create_chatbot(vectorstore)
+                            st.session_state.chatbot = chatbot
                             st.session_state.chat_history = []
-                            st.success(f"✅ Successfully processed {len(chunks)} content chunks!")
+                            
+                            if chatbot == "simple":
+                                st.warning("⚠️ Using simple chat mode (AI features limited)")
+                            else:
+                                st.success(f"✅ AI analysis ready! Processed {len(chunks) if chunks else 1} content chunks.")
                             st.balloons()
                         else:
-                            st.error("❌ Failed to process data for AI analysis")
+                            st.error("❌ Failed to process data for analysis")
                     else:
                         error_msg = extracted_data.get("error", "Unknown error occurred")
                         st.error(f"❌ Extraction failed: {error_msg}")
@@ -359,12 +451,12 @@ def main():
             if st.button("🗑️ Clear Chat History", type="secondary", use_container_width=True):
                 clear_chat_history()
         
-        # Debug info (optional)
+        # Debug info
         if st.checkbox("🔧 Show Debug Info", False):
             st.markdown("### Debug Information")
             st.write("Extracted Data:", st.session_state.extracted_data is not None)
-            st.write("Vectorstore:", st.session_state.vectorstore is not None)
-            st.write("Chatbot:", st.session_state.chatbot is not None)
+            st.write("Vectorstore Type:", type(st.session_state.vectorstore).__name__ if st.session_state.vectorstore else "None")
+            st.write("Chatbot Type:", "simple" if st.session_state.chatbot == "simple" else type(st.session_state.chatbot).__name__ if st.session_state.chatbot else "None")
             st.write("Chat History Length:", len(st.session_state.chat_history))
             st.write("Processing:", st.session_state.processing)
     
@@ -408,8 +500,6 @@ def main():
             st.info("""
             👋 **Welcome to LinkedIn AI Analyzer!**
             
-            **Powered by Mistral 7B AI**
-            
             **To get started:**
             1. Select content type
             2. Enter a LinkedIn URL or click a suggested company
@@ -421,132 +511,107 @@ def main():
             - 🏢 Company Pages  
             - 📝 Public Posts
             
-            **AI Features:**
-            - Smart content analysis
-            - Conversational chat
+            **Features:**
+            - Content extraction
+            - Basic analysis
+            - Interactive chat
             - Data insights
-            - Content summarization
             """)
     
     with col2:
         st.markdown("### 💬 AI Chat Analysis")
         
-        # Check if we have everything needed for chat
         has_extracted_data = st.session_state.extracted_data and st.session_state.extracted_data.get("status") == "success"
-        has_vectorstore = st.session_state.vectorstore is not None
-        
-        if has_extracted_data and has_vectorstore:
-            # Initialize chatbot if not exists
-            if st.session_state.chatbot is None:
-                with st.spinner("🔄 Initializing AI Chat..."):
-                    st.session_state.chatbot = create_chatbot(st.session_state.vectorstore)
-                    if st.session_state.chatbot:
-                        st.success("✅ AI Chat ready!")
-                    else:
-                        st.error("❌ Failed to initialize AI chat")
+        
+        if has_extracted_data:
+            st.success("💬 Chat ready! Ask questions about the LinkedIn data.")
             
-            # Display chat interface when chatbot is ready
-            if st.session_state.chatbot:
-                st.success("💬 Chat with Mistral AI about the LinkedIn data!")
-                
-                # Display chat history
-                for chat in st.session_state.chat_history:
-                    if chat["role"] == "user":
-                        with st.chat_message("user"):
-                            st.write(chat['content'])
-                    elif chat["role"] == "assistant":
-                        with st.chat_message("assistant"):
-                            st.write(chat['content'])
-                
-                # Chat input - ALWAYS VISIBLE when ready
-                user_input = st.chat_input("Ask about the LinkedIn data...")
+            # Display chat history
+            for chat in st.session_state.chat_history:
+                if chat["role"] == "user":
+                    with st.chat_message("user"):
+                        st.write(chat['content'])
+                elif chat["role"] == "assistant":
+                    with st.chat_message("assistant"):
+                        st.write(chat['content'])
+            
+            # Chat input
+            user_input = st.chat_input("Ask about the LinkedIn data...")
+            
+            if user_input:
+                # Add user message to history
+                st.session_state.chat_history.append({"role": "user", "content": user_input})
                 
-                if user_input:
-                    # Add user message to history and display
-                    st.session_state.chat_history.append({"role": "user", "content": user_input})
-                    
-                    # Generate AI response
-                    with st.spinner("🤔 Mistral AI is analyzing..."):
+                # Generate response based on available capabilities
+                if st.session_state.chatbot == "simple" or st.session_state.chatbot is None:
+                    # Use simple analysis
+                    with st.spinner("🤔 Analyzing..."):
+                        response = simple_chat_analysis(user_input, st.session_state.extracted_data)
+                        st.session_state.chat_history.append({"role": "assistant", "content": response})
+                        st.rerun()
+                else:
+                    # Use AI chatbot
+                    with st.spinner("🤔 AI is analyzing..."):
                         try:
                             response = st.session_state.chatbot.invoke({"question": user_input})
                             answer = response.get("answer", "I couldn't generate a response based on the available data.")
-                            
                             st.session_state.chat_history.append({"role": "assistant", "content": answer})
                             st.rerun()
                         except Exception as e:
-                            error_msg = f"❌ Error generating response: {str(e)}"
-                            st.session_state.chat_history.append({"role": "assistant", "content": error_msg})
+                            error_msg = f"❌ AI Error: {str(e)}. Using simple analysis."
+                            simple_response = simple_chat_analysis(user_input, st.session_state.extracted_data)
+                            st.session_state.chat_history.append({"role": "assistant", "content": f"{error_msg}\n\n{simple_response}"})
                             st.rerun()
-                
-                # Suggested questions - only show when no chat history
-                if len(st.session_state.chat_history) == 0:
-                    st.markdown("#### 💡 Try asking:")
-                    suggestions = [
-                        "Summarize the main information from this LinkedIn page",
-                        "What are the key highlights or achievements mentioned?",
-                        "Analyze the professional focus and expertise",
-                        "What insights can you extract from this content?",
-                        "Provide a comprehensive overview of this profile"
-                    ]
-                    
-                    for suggestion in suggestions:
-                        if st.button(suggestion, key=f"suggest_{suggestion}", use_container_width=True):
-                            st.info(f"💡 Type this in the chat: '{suggestion}'")
             
-        elif has_extracted_data and not has_vectorstore:
-            st.warning("🔄 Processing data for AI analysis...")
-            # Try to process the data
-            with st.spinner("Preparing data for AI chat..."):
-                vectorstore, chunks = process_extracted_data(st.session_state.extracted_data)
-                if vectorstore:
-                    st.session_state.vectorstore = vectorstore
-                    st.rerun()
-                else:
-                    st.error("❌ Failed to process data for AI analysis")
+            # Suggested questions
+            if len(st.session_state.chat_history) == 0:
+                st.markdown("#### 💡 Try asking:")
+                suggestions = [
+                    "Summarize the main information",
+                    "What are the key highlights?",
+                    "Analyze the professional focus",
+                    "What insights can you extract?",
+                    "Tell me about the experience"
+                ]
+                
+                for suggestion in suggestions:
+                    if st.button(suggestion, key=f"suggest_{suggestion}", use_container_width=True):
+                        st.info(f"💡 Type in chat: '{suggestion}'")
         
         elif st.session_state.processing:
             st.info("🔄 Extracting and processing LinkedIn data...")
         
         else:
-            st.info("""
-            🔍 **Extract LinkedIn data to enable AI analysis**
-            
-            Once data is extracted, you can:
-            - Ask questions about the content
-            - Get summaries and insights  
-            - Analyze professional information
-            - Extract key achievements
-            - Discuss career highlights
-            """)
+            st.info("🔍 Extract LinkedIn data to enable analysis")
 
     # Features section
     st.markdown("---")
-    st.markdown("### 🚀 Powered by Mistral 7B AI")
+    st.markdown("### 🚀 Analysis Features")
     
     feature_cols = st.columns(3)
     
     with feature_cols[0]:
         st.markdown("""
-        **🤖 Advanced AI**
-        - Mistral 7B Instruct model
-        - Intelligent text analysis
-        - Contextual understanding
+        **📊 Content Extraction**
+        - LinkedIn data scraping
+        - Text processing
+        - Content analysis
         """)
     
     with feature_cols[1]:
         st.markdown("""
         **💬 Smart Chat**
-        - Conversational memory
-        - Relevant responses
-        - Data-driven insights
+        - Interactive conversation
+        - Data-driven responses
+        - Context awareness
         """)
     
     with feature_cols[2]:
         st.markdown("""
-        **🔍 Deep Analysis**
+        **🔍 Insights**
         - Content summarization
         - Pattern recognition
-        - Professional insights
+        - Professional analysis
         """)
 
 if __name__ == "__main__":