# Install required libraries
!pip install requests beautifulsoup4 duckduckgo-search transformers aiohttp asyncio sentence-transformers

import requests
from bs4 import BeautifulSoup
from duckduckgo_search import ddg
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, pipeline
from sentence_transformers import SentenceTransformer, util
import asyncio
import aiohttp
import torch

# Set up the T5-11B model for powerful text understanding and summarization
# Note: Requires substantial resources; ensure your environment can support it
model_name = 't5-11b'  # Or 't5-3b' if resources are limited
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSeq2SeqLM.from_pretrained(model_name)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)

# Sentence-BERT model for semantic similarity
sbert_model = SentenceTransformer('all-mpnet-base-v2')  # Efficient and accurate model

# Function to perform a DuckDuckGo search and get unlimited results
def search_web(query):
    # Retrieve as many results as possible (here, up to 50)
    search_results = ddg(query, max_results=50)
    return search_results

# Asynchronous function to scrape content from a URL
async def scrape_url(session, url):
    headers = {
        "User-Agent": "MyAIChatBot/1.0 (+http://yourdomain.com/bot)"
    }
    try:
        async with session.get(url, headers=headers, timeout=15) as response:
            content_type = response.headers.get('Content-Type', '')
            if 'text/html' in content_type:
                html = await response.text()
                soup = BeautifulSoup(html, "html.parser")
                paragraphs = soup.find_all('p')
                text = "\n".join([para.get_text() for para in paragraphs])
                return text
            else:
                return ""
    except Exception:
        return ""

# Function to check if the content is relevant using semantic similarity
def is_relevant_content(user_query, content):
    # Compute embeddings
    query_embedding = sbert_model.encode(user_query, convert_to_tensor=True)
    content_embedding = sbert_model.encode(content, convert_to_tensor=True)
    # Compute cosine similarity
    similarity = util.pytorch_cos_sim(query_embedding, content_embedding).item()
    # Define a threshold for relevance (adjust as needed)
    return similarity > 0.3

# Function to extract the relevant information from the content
def extract_information(user_query, content):
    # Chunk the content to handle long texts
    max_tokens = 512  # Max tokens for the model
    sentences = content.split('. ')
    chunks = []
    current_chunk = ''
    for sentence in sentences:
        if len(tokenizer.encode(current_chunk + sentence, truncation=False)) < max_tokens:
            current_chunk += sentence + '. '
        else:
            chunks.append(current_chunk)
            current_chunk = sentence + '. '
    if current_chunk:
        chunks.append(current_chunk)

    # Process each chunk
    answers = []
    for chunk in chunks:
        prompt = f"Question: {user_query}\n\nContent: {chunk}\n\nAnswer the question based on the content."
        inputs = tokenizer.encode(prompt, return_tensors="pt", truncation=True).to(device)
        outputs = model.generate(
            inputs,
            max_length=256,
            num_beams=5,
            no_repeat_ngram_size=2,
            early_stopping=True
        )
        answer = tokenizer.decode(outputs[0], skip_special_tokens=True)
        answers.append(answer.strip())

    # Combine and return answers
    final_answer = ' '.join(answers)
    return final_answer

# Asynchronous function to handle scraping and processing of multiple URLs
async def process_urls(user_query, urls):
    tasks = []
    async with aiohttp.ClientSession() as session:
        for url in urls:
            tasks.append(scrape_url(session, url))
        contents = await asyncio.gather(*tasks)
    results = []
    for content in contents:
        if content:
            relevance = is_relevant_content(user_query, content)
            if relevance:
                information = extract_information(user_query, content)
                if information:
                    results.append(information)
    return results

# Main function for chatting
def main():
    print("Chat with AI (type 'exit' to stop)")
    while True:
        # Get user input
        user_input = input("You: ")
        
        if user_input.strip().lower() == 'exit':
            print("AI: Goodbye!")
            break

        # Treat every input as a query for specific information
        print("AI: Searching the web for the information...")
        search_results = search_web(user_input)
        urls = []
        for result in search_results:
            url = result.get('href') or result.get('url') or result.get('link')
            if url:
                urls.append(url)

        # Process URLs and get results
        loop = asyncio.get_event_loop()
        answers = loop.run_until_complete(process_urls(user_input, urls))

        if answers:
            # Combine and present the answers
            final_answer = '\n'.join(answers)
            print(f"AI: I found the following information:\n{final_answer}\n")
        else:
            print("AI: I'm sorry, I couldn't find the information you requested.")

# Run the main loop
if __name__ == "__main__":
    main()