from langchain.chat_models import ChatOpenAI from langchain.prompts import ChatPromptTemplate from langchain.schema.output_parser import StrOutputParser import requests from bs4 import BeautifulSoup from langchain.schema.runnable import RunnablePassthrough, RunnableLambda from langchain.utilities import DuckDuckGoSearchAPIWrapper import json from urllib3.exceptions import NewConnectionError from langchain_community.document_loaders import UnstructuredURLLoader from langchain_google_genai import GoogleGenerativeAI from langchain_core.prompts import PromptTemplate import os import openai import sys import gradio as gr from langchain_community.utilities import GoogleSearchAPIWrapper from langchain_core.tools import Tool from huggingface_hub import HfApi from dotenv import load_dotenv, find_dotenv _ = load_dotenv(find_dotenv()) sys.path.append('../..') os.environ["OPENAI_API_KEY"]='sk-proj-ukx0nfRmfiOHojgJAvsBT3BlbkFJhYsVcTGHeqtTdCYXZvUD' openai.api_key = os.environ["OPENAI_API_KEY"] api_key='AIzaSyAK26dJHr6PtqjxYGwb052NgIMMITbii2E' os.environ["GOOGLE_API_KEY"]="AIzaSyAK26dJHr6PtqjxYGwb052NgIMMITbii2E" os.environ["GOOGLE_CSE_ID"]="a64e58dbd396042ed" llm = GoogleGenerativeAI(model="gemini-pro", google_api_key=api_key) RESULTS_PER_QUESTION = 3 ddg_search = DuckDuckGoSearchAPIWrapper() def web_search(query: str, num_results: int = RESULTS_PER_QUESTION): results = ddg_search.results(query, num_results) return [r["link"] for r in results] def scrape_text(url: str): # Send a GET request to the webpage try: response = requests.get(url) # Check if the request was successful if response.status_code == 200: # Parse the content of the request with BeautifulSoup soup = BeautifulSoup(response.text, "html.parser") # Extract all text from the webpage page_text = soup.get_text(separator=" ", strip=True) # Print the extracted text return page_text else: return f"Failed to retrieve the webpage: Status code {response.status_code}" except Exception as e: print(e) return f"Failed to retrieve the webpage: {e}" SUMMARY_TEMPLATE = """{text} ----------- Using the above text, answer in short the following question: > {question} ----------- if the question cannot be answered using the text, imply summarize the text. Include all factual information, numbers, stats etc if available.""" SUMMARY_PROMPT = ChatPromptTemplate.from_template(SUMMARY_TEMPLATE) scrape_and_summarize_chain = RunnablePassthrough.assign( summary=RunnablePassthrough.assign( text=lambda x: scrape_text(x["url"])[:10000] ) | SUMMARY_PROMPT | llm | StrOutputParser() # Use GeminiPro with temperature adjustment ) | (lambda x: f"URL: {x['url']}\n\nSUMMARY: {x['summary']}") web_search_chain = RunnablePassthrough.assign( urls=lambda x: web_search(x["question"]) ) | (lambda x: [{"question": x["question"], "url": u} for u in x["urls"]]) | scrape_and_summarize_chain.map() SEARCH_PROMPT = ChatPromptTemplate.from_messages( [ ( "user", "Write 3 google search queries to search online that form an " "objective opinion from the following: {question}\n" "You must respond with a list of strings in the following format: " '["query 1", "query 2", "query 3"].', ), ] ) search_question_chain = SEARCH_PROMPT | llm | StrOutputParser() | json.loads full_research_chain = search_question_chain | (lambda x: [{"question": q} for q in x]) | web_search_chain.map() WRITER_SYSTEM_PROMPT = "You are an AI Shopping Assistants. Your sole purpose is to write well written, critically acclaimed, objective and structured reports on given text." # noqa: E501 # Report prompts from https://github.com/assafelovic/gpt-researcher/blob/master/gpt_researcher/master/prompts.py RESEARCH_REPORT_TEMPLATE = """Information: -------- {research_summary} -------- Using the above information, answer the following question or topic: "{question}" in a detailed report -- \ The report should focus on the answer to the question, should be well structured, informative, \ in depth, with facts,baterry and processor benchmarks,prices of the products in all stores and Write price in each store and write the name of the store and check if there were any offers and a minimum of 200 words. You should compare between the product in question and products in the same price range. You should suggest the best products based on price range in the question and compare between them. You should strive to write the report as long as you can using all relevant and necessary information provided. You must write the report with markdown syntax. You MUST determine your own concrete and valid opinion based on the given information. Do NOT deter to general and meaningless conclusions. Write all used source urls at the end of the report, and make sure to not add duplicated sources, but only one reference for each. You must write the report in apa format. Please do your best, this is very important to my opinion about product.""" prompt = ChatPromptTemplate.from_messages( [ ("system", WRITER_SYSTEM_PROMPT), ("user", RESEARCH_REPORT_TEMPLATE), ] ) def collapse_list_of_lists(list_of_lists): content = [] for l in list_of_lists: content.append("\n\n".join(l)) return "\n\n".join(content) chain = RunnablePassthrough.assign( research_summary= full_research_chain | collapse_list_of_lists ) | prompt | llm | StrOutputParser() def predict(input, history): response= chain.invoke({"question":input}) return str(response) gr.ChatInterface(predict,description="These assistants could help customers find products, compare options,answer questions and recommend the best product based on customer needs.").launch(share=True)