Rename services.py to web_scraper.py

services.py

@@ -1,111 +0,0 @@
-# /services.py
-""" Manages interactions with all external LLM and search APIs. """
-
-import os
-import logging
-from typing import Dict, Any, Generator, List
-
-from dotenv import load_dotenv
-from huggingface_hub import InferenceClient
-from tavily import TavilyClient
-from groq import Groq
-import fireworks.client as Fireworks
-import openai
-import google.generativeai as genai
-
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-load_dotenv()
-
-# --- API Keys from .env ---
-HF_TOKEN = os.getenv("HF_TOKEN")
-TAVILY_API_KEY = os.getenv("TAVILY_API_KEY")
-GROQ_API_KEY = os.getenv("GROQ_API_KEY")
-FIREWORKS_API_KEY = os.getenv("FIREWORKS_API_KEY")
-OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
-GEMINI_API_KEY = os.getenv("GEMINI_API_KEY")
-DEEPSEEK_API_KEY = os.getenv("DEEPSEEK_API_KEY")
-
-Messages = List[Dict[str, Any]]
-
-class LLMService:
-    """A multi-provider wrapper for LLM Inference APIs."""
-    def __init__(self):
-        self.hf_client = InferenceClient(token=HF_TOKEN) if HF_TOKEN else None
-        self.groq_client = Groq(api_key=GROQ_API_KEY) if GROQ_API_KEY else None
-        self.openai_client = openai.OpenAI(api_key=OPENAI_API_KEY) if OPENAI_API_KEY else None
-        
-        if DEEPSEEK_API_KEY:
-            self.deepseek_client = openai.OpenAI(api_key=DEEPSEEK_API_KEY, base_url="https://api.deepseek.com/v1")
-        else:
-            self.deepseek_client = None
-
-        if FIREWORKS_API_KEY:
-            Fireworks.api_key = FIREWORKS_API_KEY
-            self.fireworks_client = Fireworks
-        else:
-            self.fireworks_client = None
-
-        if GEMINI_API_KEY:
-            genai.configure(api_key=GEMINI_API_KEY)
-            self.gemini_model = genai.GenerativeModel('gemini-1.5-pro-latest')
-        else:
-            self.gemini_model = None
-
-    def _prepare_messages_for_gemini(self, messages: Messages) -> List[Dict[str, Any]]:
-        gemini_messages = []
-        for msg in messages:
-            if msg['role'] == 'system': continue # Gemini doesn't use a system role in this way
-            role = 'model' if msg['role'] == 'assistant' else 'user'
-            gemini_messages.append({'role': role, 'parts': [msg['content']]})
-        return gemini_messages
-
-    def generate_code_stream(self, model_id: str, messages: Messages, max_tokens: int = 8192) -> Generator[str, None, None]:
-        provider, model_name = model_id.split('/', 1)
-        logging.info(f"Dispatching to provider: {provider} for model: {model_name}")
-
-        try:
-            if provider in ['openai', 'groq', 'deepseek', 'fireworks']:
-                client_map = {'openai': self.openai_client, 'groq': self.groq_client, 'deepseek': self.deepseek_client, 'fireworks': self.fireworks_client.ChatCompletion if self.fireworks_client else None}
-                client = client_map.get(provider)
-                if not client: raise ValueError(f"{provider.capitalize()} API key not configured.")
-                
-                stream = client.create(model=model_name, messages=messages, stream=True, max_tokens=max_tokens) if provider == 'fireworks' else client.chat.completions.create(model=model_name, messages=messages, stream=True, max_tokens=max_tokens)
-                for chunk in stream:
-                    if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.content: yield chunk.choices[0].delta.content
-            
-            elif provider == 'gemini':
-                if not self.gemini_model: raise ValueError("Gemini API key not configured.")
-                system_prompt = next((msg['content'] for msg in messages if msg['role'] == 'system'), "")
-                gemini_messages = self._prepare_messages_for_gemini(messages)
-                # Prepend system prompt to first user message for Gemini
-                if system_prompt and gemini_messages and gemini_messages[0]['role'] == 'user':
-                    gemini_messages[0]['parts'][0] = f"{system_prompt}\n\n{gemini_messages[0]['parts'][0]}"
-                stream = self.gemini_model.generate_content(gemini_messages, stream=True)
-                for chunk in stream: yield chunk.text
-
-            elif provider == 'huggingface':
-                if not self.hf_client: raise ValueError("Hugging Face API token not configured.")
-                hf_model_id = model_id.split('/', 1)[1]
-                stream = self.hf_client.chat_completion(model=hf_model_id, messages=messages, stream=True, max_tokens=max_tokens)
-                for chunk in stream:
-                    if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.content: yield chunk.choices[0].delta.content
-            else:
-                raise ValueError(f"Unknown provider: {provider}")
-        except Exception as e:
-            logging.error(f"LLM API Error with provider {provider}: {e}")
-            yield f"Error from {provider.capitalize()}: {str(e)}"
-
-class SearchService:
-    def __init__(self, api_key: str = TAVILY_API_KEY):
-        self.client = TavilyClient(api_key=api_key) if api_key else None
-        if not self.client: logging.warning("TAVILY_API_KEY not set. Web search will be disabled.")
-    def is_available(self) -> bool: return self.client is not None
-    def search(self, query: str, max_results: int = 5) -> str:
-        if not self.is_available(): return "Web search is not available."
-        try:
-            response = self.client.search(query, search_depth="advanced", max_results=min(max(1, max_results), 10))
-            return "Web Search Results:\n\n" + "\n---\n".join([f"Title: {res.get('title', 'N/A')}\nURL: {res.get('url', 'N/A')}\nContent: {res.get('content', 'N/A')}" for res in response.get('results', [])])
-        except Exception as e: return f"Search error: {str(e)}"
-
-llm_service = LLMService()
-search_service = SearchService()

web_scraper.py

@@ -0,0 +1,237 @@
+def extract_text_from_image(image_path):
+    """Extract text from image using OCR"""
+    try:
+        try:
+            pytesseract.get_tesseract_version()
+        except Exception:
+            return "Error: Tesseract OCR is not installed. Please install Tesseract to extract text from images. See install_tesseract.md for instructions."
+        image = cv2.imread(image_path)
+        if image is None:
+            return "Error: Could not read image file"
+        image_rgb=cv2.cvtColor(image,cv2.COLOR_BGR2RGB)
+        gray=cv2.cvtColor(image_rgb,cv2.COLOR_RGB2GRAY)
+        _,binary=cv2.threshold(gray,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)
+        text=pytesseract.image_to_string(binary,config='--psm 6')
+        return text.strip() if text.strip() else "No text found in image"
+    except Exception as e:
+        return f"Error extracting text from image: {e}"
+
+def extract_text_from_file(file_path):
+    if not file_path:
+        return ""
+    mime,_=mimetypes.guess_type(file_path)
+    ext=os.path.splitext(file_path)[1].lower()
+    try:
+        if ext==".pdf":
+            with open(file_path,"rb") as f:
+                reader=PyPDF2.PdfReader(f)
+                return "\n".join(page.extract_text() or "" for page in reader.pages)
+        elif ext in [".txt", ".md"]:
+            with open(file_path,"r",encoding="utf-8") as f:
+                return f.read()
+        elif ext==".csv":
+            with open(file_path,"r",encoding="utf-8") as f:
+                return f.read()
+        elif ext==".docx":
+            doc=docx.Document(file_path)
+            return "\n".join([para.text for para in doc.paragraphs])
+        elif ext.lower() in [".jpg",".jpeg",".png",".bmp",".tiff",".tif",".gif",".webp"]:
+            return extract_text_from_image(file_path)
+        else:
+            return ""
+    except Exception as e:
+        return f"Error extracting text: {e}"
+
+def extract_website_content(url: str) -> str:
+    """Extract HTML code and content from a website URL"""
+    try:
+        parsed_url=urlparse(url)
+        if not parsed_url.scheme:
+            url="https://"+url
+            parsed_url=urlparse(url)
+        if not parsed_url.netloc:
+            return "Error: Invalid URL provided"
+        headers={
+            'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+            'Accept-Language':'en-US,en;q=0.9',
+            'Accept-Encoding':'gzip, deflate, br',
+            'DNT':'1','Connection':'keep-alive','Upgrade-Insecure-Requests':'1',
+            'Sec-Fetch-Dest':'document','Sec-Fetch-Mode':'navigate','Sec-Fetch-Site':'none','Sec-Fetch-User':'?1','Cache-Control':'max-age=0'
+        }
+        session=requests.Session()
+        session.headers.update(headers)
+        max_retries=3
+        for attempt in range(max_retries):
+            try:
+                response=session.get(url,timeout=15,allow_redirects=True)
+                response.raise_for_status()
+                break
+            except requests.exceptions.HTTPError as e:
+                if e.response.status_code==403 and attempt<max_retries-1:
+                    session.headers['User-Agent']='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+                    continue
+                else:
+                    raise
+        try:
+            response.encoding=response.apparent_encoding
+            raw_html=response.text
+        except:
+            raw_html=response.content.decode('utf-8',errors='ignore')
+        if not raw_html.strip().startswith('<!DOCTYPE') and not raw_html.strip().startswith('<html'):
+            try:
+                raw_html=response.content.decode('latin-1',errors='ignore')
+            except:
+                try:
+                    raw_html=response.content.decode('utf-8',errors='ignore')
+                except:
+                    raw_html=response.content.decode('cp1252',errors='ignore')
+        soup=BeautifulSoup(raw_html,'html.parser')
+        title=soup.find('title')
+        title_text=title.get_text().strip() if title else "No title found"
+        meta_desc=soup.find('meta',attrs={'name':'description'})
+        description=meta_desc.get('content','') if meta_desc else ""
+        content_sections=[]
+        main_selectors=['main','article','.content','.main-content','.post-content','#content','#main','.entry-content','.post-body']
+        for selector in main_selectors:
+            elements=soup.select(selector)
+            for element in elements:
+                text=element.get_text().strip()
+                if len(text)>100:
+                    content_sections.append(text)
+        nav_links=[]
+        nav_elements=soup.find_all(['nav','header'])
+        for nav in nav_elements:
+            links=nav.find_all('a')
+            for link in links:
+                link_text=link.get_text().strip()
+                link_href=link.get('href','')
+                if link_text and link_href:
+                    nav_links.append(f"{link_text}: {link_href}")
+        img_elements=soup.find_all('img')
+        for img in img_elements:
+            src=img.get('src','')
+            if src:
+                if src.startswith('//'):
+                    absolute_src='https:'+src
+                    img['src']=absolute_src
+                elif src.startswith('/'):
+                    absolute_src=urljoin(url,src)
+                    img['src']=absolute_src
+                elif not src.startswith(('http://','https://')):
+                    absolute_src=urljoin(url,src)
+                    img['src']=absolute_src
+                data_src=img.get('data-src','')
+                if data_src and not src:
+                    if data_src.startswith('//'):
+                        absolute_data_src='https:'+data_src
+                        img['src']=absolute_data_src
+                    elif data_src.startswith('/'):
+                        absolute_data_src=urljoin(url,data_src)
+                        img['src']=absolute_data_src
+                    elif not data_src.startswith(('http://','https://')):
+                        absolute_data_src=urljoin(url,data_src)
+                        img['src']=absolute_data_src
+                    else:
+                        img['src']=data_src
+        elements_with_style=soup.find_all(attrs={'style':True})
+        for element in elements_with_style:
+            style_attr=element.get('style','')
+            import re
+            bg_pattern=r'background-image:\s*url\(["\']?([^"\']+)["\']?\)'
+            matches=re.findall(bg_pattern,style_attr, re.IGNORECASE)
+            for match in matches:
+                if match.startswith('//'):
+                    absolute_bg='https:'+match
+                    style_attr=style_attr.replace(match,absolute_bg)
+                elif match.startswith('/'):
+                    absolute_bg=urljoin(url,match)
+                    style_attr=style_attr.replace(match,absolute_bg)
+                elif not match.startswith(('http://','https://')):
+                    absolute_bg=urljoin(url,match)
+                    style_attr=style_attr.replace(match,absolute_bg)
+            element['style']=style_attr
+        style_elements=soup.find_all('style')
+        for style in style_elements:
+            if style.string:
+                style_content=style.string
+                bg_pattern=r'background-image:\s*url\(["\']?([^"\']+)["\']?\)'
+                matches=re.findall(bg_pattern,style_content, re.IGNORECASE)
+                for match in matches:
+                    if match.startswith('//'):
+                        absolute_bg='https:'+match
+                        style_content=style_content.replace(match,absolute_bg)
+                    elif match.startswith('/'):
+                        absolute_bg=urljoin(url,match)
+                        style_content=style_content.replace(match,absolute_bg)
+                    elif not match.startswith(('http://','https://')):
+                        absolute_bg=urljoin(url,match)
+                        style_content=style_content.replace(match,absolute_bg)
+                style.string=style_content
+        images=[]
+        img_elements=soup.find_all('img')
+        for img in img_elements:
+            src=img.get('src','')
+            alt=img.get('alt','')
+            if src:
+                images.append({'src':src,'alt':alt})
+        def test_image_url(img_url):
+            try:
+                test_response=requests.head(img_url,timeout=5,allow_redirects=True)
+                return test_response.status_code==200
+            except:
+                return False
+        working_images=[]
+        for img in images[:10]:
+            if test_image_url(img['src']):
+                working_images.append(img)
+        modified_html=str(soup)
+        import re
+        cleaned_html=re.sub(r'<!--.*?-->','',modified_html,flags=re.DOTALL)
+        cleaned_html=re.sub(r'\s+',' ',cleaned_html)
+        cleaned_html=re.sub(r'>\s+<','><',cleaned_html)
+        if len(cleaned_html)>15000:
+            cleaned_html=cleaned_html[:15000]+"\n<!-- ... HTML truncated for length ... -->"
+        if not title_text or title_text=="No title found":
+            title_text=url.split('/')[-1] or url.split('/')[-2] or "Website"
+        if len(cleaned_html.strip())<100:
+            website_content=f"""
+WEBSITE REDESIGN - EXTRACTION FAILED
+====================================
+URL: {url}
+Title: {title_text}
+ERROR: Could not extract meaningful HTML content from this website. This could be due to:
+1. The website uses heavy JavaScript to load content dynamically
+2. The website has anti-bot protection
+3. The website requires authentication
+4. The website is using advanced compression or encoding
+FALLBACK APPROACH:
+Please create a modern, responsive website design for a {title_text.lower()} website. Since I couldn't extract the original content, you can:
+1. Create a typical layout for this type of website
+2. Use placeholder content that would be appropriate
+3. Include modern design elements and responsive features
+4. Use a clean, professional design with good typography
+5. Make it mobile-friendly and accessible
+This will help me create a better design for you."""
+            return website_content.strip()
+        website_content=f"""
+WEBSITE REDESIGN - ORIGINAL HTML CODE
+===[TRUNCATED FOR BREVITY]==="""
+        return website_content.strip()
+    except requests.exceptions.HTTPError as e:
+        if e.response.status_code==403:
+            return f"Error: Website blocked access (403 Forbidden). This website may have anti-bot protection. Try a different website or provide a description of what you want to build instead."
+        elif e.response.status_code==404:
+            return f"Error: Website not found (404). Please check the URL and try again."
+        elif e.response.status_code>=500:
+            return f"Error: Website server error ({e.response.status_code}). Please try again later."
+        else:
+            return f"Error accessing website: HTTP {e.response.status_code} - {str(e)}"
+    except requests.exceptions.Timeout:
+        return "Error: Request timed out. The website may be slow or unavailable."
+    except requests.exceptions.ConnectionError:
+        return "Error: Could not connect to the website. Please check your internet connection and the URL."
+    except requests.exceptions.RequestException as e:
+        return f"Error accessing website: {str(e)}"
+    except Exception as e:
+        return f"Error extracting website content: {str(e)}"