app.py

import os
import gradio as gr
import cohere
import requests
from crewai import Agent, Task, Crew, Process

from langchain_groq import ChatGroq
from langchain_cohere import ChatCohere

from langchain_community.tools import DuckDuckGoSearchRun, DuckDuckGoSearchResults
from crewai_tools import tool, SeleniumScrapingTool, ScrapeWebsiteTool
from duckduckgo_search import DDGS

from newspaper import Article

# Ensure essential environment variables are set
cohere_api_key = os.getenv('COHERE_API_KEY')
if not cohere_api_key:
    raise EnvironmentError("COHERE_API_KEY is not set in environment variables")
groq_api_key = os.getenv("GROQ_API_KEY")
if not groq_api_key:
    raise EnvironmentError("GROQ_API_KEY is not set in environment variables")

# Initialize API clients
co = cohere.Client(cohere_api_key)
print("client ok")

def fetch_content(url):
    try:
        article = Article(url)
        article.download()
        article.parse()
        return article.text
    except Exception as e:
        print("ERROR: " + str(e))
        return f"Error fetching content: {e}"

# Define the DuckDuckGoSearch tool
@tool('DuckDuckGoSearchResults')
def search_results(search_query: str) -> dict:
    """
    Performs a web search to gather and return a collection of search results.
    This tool automates the retrieval of web-based information related to a specified query.
    Args:
    - search_query (str): The query string that specifies the information to be searched on the web. This should be a clear and concise expression of the user's information needs.
    Returns:
    - list: A list of dictionaries, where each dictionary represents a search result. Each dictionary includes 'snippet' of the page and the 'link' with the url linking to it.
    """
    results = DDGS().text(search_query, max_results=5, timelimit='m')
    results_list = [{"title": result['title'], "snippet": result['body'], "link": result['href']} for result in results]
    return results_list

@tool('WebScrapper')
def web_scrapper(url: str, topic: str) -> str:
    """
    A tool designed to extract and read the content of a specified link and generate a summary on a specific topic.
    It is capable of handling various types of web pages by making HTTP requests and parsing the received HTML content.
    This tool is particularly useful for web scraping tasks, data collection, or extracting specific information from websites.
    
    Args:
    - url (str): The URL from which to scrape content.
    - topic (str): The specific topic on which to generate a summary.
    Returns:
    - summary (str): summary of the url on the topic
    """
    # Scrape content from the specified URL
    content = fetch_content(url)
    
    # Prepare the prompt for generating the summary
    prompt = f"Generate a summary of the following content on the topic ## {topic} ### \n\nCONTENT:\n\n" + content
    
    # Generate the summary using Cohere
    response = co.chat(
        model='command-r-plus',
        message=prompt,
        temperature=0.4,
        max_tokens=1000,
        chat_history=[],
        prompt_truncation='AUTO'
    )

    summary_response = f"""###
    Summary:
    {response.text}
    
    URL: {url}
    ###
    """
    
    return summary_response

def kickoff_crew(topic: str, model_choice: str) -> str:
    try:
    
        # Initialize the large language models based on user selection
        groq_llm = ChatGroq(temperature=0, groq_api_key=groq_api_key, model_name=model_choice)
    
        # Define Agents with Groq LLM
        researcher = Agent(
            role='Researcher',
            goal='Search and Collect detailed information on topic ## {topic} ##',
            tools=[search_results, web_scrapper],
            llm=groq_llm,  # Assigning the LLM here
            backstory=(
                "You are a meticulous researcher, skilled at navigating vast amounts of information to extract essential insights on any given topic. "
                "Your dedication to detail ensures the reliability and thoroughness of your findings. "
                "With a strategic approach, you carefully analyze and document data, aiming to provide accurate and trustworthy results."
            ),
            allow_delegation=False,
            max_iter=15,
            max_rpm=20,
            memory=True,
            verbose=True
        )

        
        editor = Agent(
            role='Editor',
            goal='Compile and refine the information into a comprehensive report on topic ## {topic} ##',
            llm=groq_llm,  # Assigning the LLM here
            backstory=(
                "As an expert editor, you specialize in transforming raw data into clear, engaging reports. "
                "Your strong command of language and attention to detail ensure that each report not only conveys essential insights "
                "but is also easily understandable and appealing to diverse audiences. "
            ),
            allow_delegation=False,
            max_iter=5,
            max_rpm=15,
            memory=True,
            verbose=True
        )
        
        # Define Tasks
        research_task = Task(
            description=(
                "Use the DuckDuckGoSearchResults tool to collect initial search snippets on ## {topic} ##. "
                "If more detailed searches are required, generate and execute new queries related to ## {topic} ##. "
                "Subsequently, employ the WebScrapper tool to delve deeper into significant URLs identified from the snippets, extracting further information and insights. "
                "Compile these findings into a preliminary draft, documenting all relevant sources, titles, and links associated with the topic. "
                "Ensure high accuracy throughout the process and avoid any fabrication or misrepresentation of information."
            ),
            expected_output=(
                "A structured draft report about the topic, featuring an introduction, a detailed main body organized by different aspects of the topic, and a conclusion. "
                "Each section should properly cite sources, providing a thorough overview of the information gathered."
            ),
            agent=researcher
        )

        
        edit_task = Task(
            description=(
                "Review and refine the initial draft report from the research task. Organize the content logically to enhance information flow. "
                "Verify the accuracy of all data, correct discrepancies, and update information to ensure it reflects current knowledge and is well-supported by sources. "
                "Improve the report’s readability by enhancing language clarity, adjusting sentence structures, and maintaining a consistent tone. "
                "Include a section listing all sources used, formatted as bullet points following this template: "
                "- title: url'."
            ),
            expected_output=(
                "A polished, comprehensive report on topic ## {topic} ##, with a clear, professional narrative that accurately reflects the research findings. "
                "The report should include an introduction, an extensive discussion section, a concise conclusion, and a well-organized source list. "
                "Ensure the document is grammatically correct and ready for publication or presentation."
            ),
            agent=editor,
            context=[research_task]
        )
    
        # Forming the Crew
        crew = Crew(
            agents=[researcher, editor],
            tasks=[research_task, edit_task],
            process=Process.sequential,
        )
    
        # Kick-off the research process
        result = crew.kickoff(inputs={'topic': topic})
        if not isinstance(result, str):
            result = str(result)
        return result
    except Exception as e:
        return f"Error: {str(e)}"

def main():
    """Set up the Gradio interface for the CrewAI Research Tool."""
    with gr.Blocks() as demo:
        gr.Markdown("## CrewAI Research Tool")
        topic_input = gr.Textbox(label="Enter Topic", placeholder="Type here...")
        model_choice = gr.Radio(choices=["llama3-8b-8192", "llama3-70b-8192", 'llama-3.1-8b-instant', 'llama-3.1-70b-versatile'], label="Choose Model")
        submit_button = gr.Button("Start Research")
        output = gr.Markdown(label="Result")

        submit_button.click(
            fn=kickoff_crew,
            inputs=[topic_input, model_choice],
            outputs=output
        )

    # demo.launch(debug=True)
    demo.queue(api_open=False, max_size=3).launch()

if __name__ == "__main__":
    main()