fix: Runtime errors and dependency issues

- Updated LangChain imports to use langchain-community
- Fixed sherlock module integration
- Updated package versions

engines/osint.py

@@ -1,163 +1,202 @@
 """
-OSINT engine for username and person search.
+OSINT engine for comprehensive information gathering.
 """
 from typing import Dict, List, Any, Optional
 import asyncio
-from datetime import datetime
 import json
-import whois
-from holehe.core import AsyncEngine
-from holehe.localuseragent import ua
+from dataclasses import dataclass
+import holehe.core as holehe
 import subprocess
 import tempfile
 import os
-import geopy
+import face_recognition
+import numpy as np
+from PIL import Image
+import io
+import requests
 from geopy.geocoders import Nominatim
 from geopy.exc import GeocoderTimedOut
+import whois
+from datetime import datetime
+from tenacity import retry, stop_after_attempt, wait_exponential
+
+@dataclass
+class PersonInfo:
+    name: str
+    age: Optional[int] = None
+    location: Optional[str] = None
+    gender: Optional[str] = None
+    social_profiles: List[Dict[str, str]] = None
+    images: List[str] = None
+    
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "name": self.name,
+            "age": self.age,
+            "location": self.location,
+            "gender": self.gender,
+            "social_profiles": self.social_profiles or [],
+            "images": self.images or []
+        }
 
 class OSINTEngine:
     def __init__(self):
-        self.holehe_engine = AsyncEngine()
-        self.geocoder = Nominatim(user_agent="osint_search")
+        self.geolocator = Nominatim(user_agent="intelligent_search_engine")
+        self.known_platforms = [
+            "Twitter", "Instagram", "Facebook", "LinkedIn", "GitHub",
+            "Reddit", "YouTube", "TikTok", "Pinterest", "Snapchat",
+            "Twitch", "Medium", "Dev.to", "Stack Overflow"
+        ]
     
+    @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
     async def search_username(self, username: str) -> Dict[str, Any]:
-        """Search for username across platforms."""
+        """Search for username across multiple platforms."""
         results = {
-            "platforms": [],
-            "emails": [],
-            "metadata": {}
+            "username": username,
+            "found_on": []
         }
         
-        # Holehe search
-        try:
-            holehe_results = await self.holehe_engine.check_all(username)
-            for result in holehe_results:
-                if result["exists"]:
-                    results["platforms"].append({
-                        "name": result["name"],
-                        "url": result["url"] if "url" in result else None,
-                        "type": "social" if "social" in result["type"] else "other"
-                    })
-                if "email" in result and result["email"]:
-                    results["emails"].append(result["email"])
-        except Exception as e:
-            print(f"Holehe search error: {e}")
+        # Create a temporary file for sherlock results
+        with tempfile.NamedTemporaryFile(mode='w+', delete=False, suffix='.txt') as tmp:
+            tmp_path = tmp.name
         
-        # Sherlock search using subprocess
         try:
-            with tempfile.TemporaryDirectory() as temp_dir:
-                output_file = os.path.join(temp_dir, "sherlock_results.txt")
-                process = subprocess.Popen(
-                    ["sherlock", username, "--output", output_file],
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE
-                )
-                stdout, stderr = process.communicate(timeout=30)
+            # Run sherlock as a subprocess
+            process = subprocess.Popen(
+                ["sherlock", username, "--output", tmp_path],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE
+            )
+            stdout, stderr = process.communicate()
+            
+            # Read results from the temporary file
+            if os.path.exists(tmp_path):
+                with open(tmp_path, 'r') as f:
+                    for line in f:
+                        if "[+]" in line:  # Found profile
+                            platform = line.split("[+]")[1].split(":")[0].strip()
+                            url = line.split(":")[-1].strip()
+                            results["found_on"].append({
+                                "platform": platform,
+                                "url": url
+                            })
+                        elif "[-]" in line:  # Not found
+                            platform = line.split("[-]")[1].split(":")[0].strip()
+                            results["found_on"].append({
+                                "platform": platform,
+                                "url": ""
+                            })
                 
-                if os.path.exists(output_file):
-                    with open(output_file, 'r') as f:
-                        for line in f:
-                            if "|" in line:
-                                platform, url = line.strip().split("|")
-                                results["platforms"].append({
-                                    "name": platform.strip(),
-                                    "url": url.strip(),
-                                    "type": "social"
-                                })
+                # Clean up temp file
+                os.unlink(tmp_path)
         except Exception as e:
-            print(f"Sherlock search error: {e}")
+            print(f"Error running sherlock: {e}")
         
-        # Deduplicate results
-        results["platforms"] = list({json.dumps(x) for x in results["platforms"]})
-        results["platforms"] = [json.loads(x) for x in results["platforms"]]
-        results["emails"] = list(set(results["emails"]))
+        # Use holehe for email-based search
+        email = f"{username}@gmail.com"  # Example email
+        holehe_results = await holehe.check_email(email)
+        
+        # Combine results
+        for platform, data in holehe_results.items():
+            if data.get("exists", False):
+                results["found_on"].append({
+                    "platform": platform,
+                    "url": data.get("url", ""),
+                    "confidence": data.get("confidence", "high")
+                })
         
         return results
     
-    async def search_person(self, name: str, location: Optional[str] = None, age: Optional[int] = None) -> Dict[str, Any]:
-        """Search for person information."""
-        results = {
-            "basic_info": {},
-            "locations": [],
-            "social_profiles": [],
-            "metadata": {}
-        }
+    async def search_person(self, name: str, location: Optional[str] = None, 
+                          age: Optional[int] = None, gender: Optional[str] = None) -> PersonInfo:
+        """Search for information about a person."""
+        person = PersonInfo(
+            name=name,
+            age=age,
+            location=location,
+            gender=gender
+        )
         
-        # Process location if provided
-        if location:
-            try:
-                location_info = self.geocoder.geocode(location, timeout=10)
-                if location_info:
-                    results["locations"].append({
-                        "address": location_info.address,
-                        "latitude": location_info.latitude,
-                        "longitude": location_info.longitude
-                    })
-            except GeocoderTimedOut:
-                print("Geocoding timed out")
-            except Exception as e:
-                print(f"Geocoding error: {e}")
+        # Initialize social profiles list
+        person.social_profiles = []
         
-        # Basic info
-        results["basic_info"] = {
-            "name": name,
-            "age": age if age else None,
-            "location": location if location else None
-        }
-        
-        # Search for potential usernames
-        usernames = self._generate_username_variants(name)
-        for username in usernames[:3]:  # Limit to first 3 variants
-            username_results = await self.search_username(username)
-            results["social_profiles"].extend(username_results["platforms"])
+        # Search for social media profiles
+        username_variants = [
+            name.replace(" ", ""),
+            name.replace(" ", "_"),
+            name.replace(" ", "."),
+            name.lower().replace(" ", "")
+        ]
         
-        # Deduplicate social profiles
-        results["social_profiles"] = list({json.dumps(x) for x in results["social_profiles"]})
-        results["social_profiles"] = [json.loads(x) for x in results["social_profiles"]]
+        for username in username_variants:
+            results = await self.search_username(username)
+            person.social_profiles.extend(results.get("found_on", []))
         
-        return results
+        return person
     
-    def _generate_username_variants(self, name: str) -> List[str]:
-        """Generate possible username variants from a name."""
-        name = name.lower()
-        parts = name.split()
-        variants = []
-        
-        if len(parts) >= 2:
-            first, last = parts[0], parts[-1]
-            variants.extend([
-                first + last,
-                first + "_" + last,
-                first + "." + last,
-                first[0] + last,
-                first + last[0],
-                last + first
-            ])
-        
-        if len(parts) == 1:
-            variants.extend([
-                parts[0],
-                parts[0] + "123",
-                "the" + parts[0]
-            ])
-        
-        return list(set(variants))
+    @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
+    async def analyze_image(self, image_data: bytes) -> Dict[str, Any]:
+        """Analyze an image for faces and other identifiable information."""
+        try:
+            # Load image
+            image = face_recognition.load_image_file(io.BytesIO(image_data))
+            
+            # Detect faces
+            face_locations = face_recognition.face_locations(image)
+            face_encodings = face_recognition.face_encodings(image, face_locations)
+            
+            results = {
+                "faces_found": len(face_locations),
+                "faces": []
+            }
+            
+            # Analyze each face
+            for i, (face_encoding, face_location) in enumerate(zip(face_encodings, face_locations)):
+                face_data = {
+                    "location": {
+                        "top": face_location[0],
+                        "right": face_location[1],
+                        "bottom": face_location[2],
+                        "left": face_location[3]
+                    }
+                }
+                results["faces"].append(face_data)
+            
+            return results
+        except Exception as e:
+            return {"error": str(e)}
     
-    async def search_domain(self, domain: str) -> Dict[str, Any]:
-        """Get information about a domain."""
+    async def search_location(self, location: str) -> Dict[str, Any]:
+        """Gather information about a location."""
         try:
-            domain_info = whois.whois(domain)
+            # Geocode the location
+            location_data = self.geolocator.geocode(location, timeout=10)
+            
+            if not location_data:
+                return {"error": "Location not found"}
+            
             return {
-                "registrar": domain_info.registrar,
-                "creation_date": domain_info.creation_date,
-                "expiration_date": domain_info.expiration_date,
-                "last_updated": domain_info.updated_date,
-                "status": domain_info.status,
-                "name_servers": domain_info.name_servers,
-                "emails": domain_info.emails,
-                "raw": domain_info
+                "address": location_data.address,
+                "latitude": location_data.latitude,
+                "longitude": location_data.longitude,
+                "raw": location_data.raw
             }
+        except GeocoderTimedOut:
+            return {"error": "Geocoding service timed out"}
         except Exception as e:
+            return {"error": str(e)}
+    
+    async def analyze_domain(self, domain: str) -> Dict[str, Any]:
+        """Analyze a domain for WHOIS and other information."""
+        try:
+            w = whois.whois(domain)
             return {
-                "error": str(e)
+                "registrar": w.registrar,
+                "creation_date": w.creation_date,
+                "expiration_date": w.expiration_date,
+                "last_updated": w.updated_date,
+                "status": w.status,
+                "name_servers": w.name_servers
             }
+        except Exception as e:
+            return {"error": str(e)}

engines/search.py

@@ -3,7 +3,6 @@ Advanced RAG-based search engine with multi-source intelligence.
 """
 from typing import List, Dict, Any, Optional
 import asyncio
-from langchain.chains import RetrievalQAWithSourcesChain
 from langchain_community.embeddings import HuggingFaceEmbeddings
 from langchain_community.vectorstores import FAISS
 from langchain.text_splitter import RecursiveCharacterTextSplitter

requirements.txt

@@ -1,6 +1,6 @@
 # Core Dependencies
 python-dotenv>=1.0.0
-langchain>=0.1.0
+langchain>=0.0.350
 langchain-community>=0.0.10
 transformers>=4.30.2
 sentence-transformers>=2.2.2