Spaces:
Runtime error
Runtime error
File size: 6,462 Bytes
52e3677 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 |
import json
from dotenv import load_dotenv
import gradio as gr
import os
from PyPDF2 import PdfReader
from langchain.text_splitter import CharacterTextSplitter
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.vectorstores import FAISS
from langchain.chains.question_answering import load_qa_chain
from langchain.chat_models import ChatOpenAI
from langchain.callbacks import get_openai_callback
from requests.exceptions import Timeout
import requests
from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin
import time
import random
import os
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
load_dotenv()
knowledge_base = None
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
}
def get_internal_links(url):
print('start get internal links')
internal_links = []
domain = urlparse(url).netloc # 获取当前网站域名
response = requests.get(url, headers=headers, timeout=5)
soup = BeautifulSoup(response.content, 'html.parser')
for a in soup.find_all('a', href=True):
href = a['href']
if href.startswith('http'): # 外链
if urlparse(href).netloc == domain: # 如果是本站链接
internal_links.append(href)
else: # 内链
internal_link = urljoin(url, href)
if urlparse(internal_link).netloc == domain:
internal_links.append(internal_link)
internal_links = list(set(internal_links))
print(internal_links)
return internal_links
def get_page_content(url):
response = requests.get(url, headers=headers, timeout=5)
soup = BeautifulSoup(response.content, 'html.parser')
content = soup.get_text()
time.sleep(random.randint(1, 3))
return content
def crawl_site(url):
links_to_visit = get_internal_links(url)
content = ""
while links_to_visit:
link = links_to_visit.pop(0)
content += get_page_content(link)
print(f'Page content for {link}:\n')
return content
def decode_pdf(file_path):
encodings = ['utf-8', 'gbk', 'gb2312', 'big5', 'cp1252'] # 常见编码方式
text = ""
with open(file_path, 'rb') as f:
pdf_reader = PdfReader(f)
for encoding in encodings:
try:
for page in pdf_reader.pages:
temp_text = page.extract_text()
encode_temp_text = temp_text.encode(encoding)
decode_temp_text = encode_temp_text.decode(encoding,'strict')
text += decode_temp_text
break
except UnicodeDecodeError:
pass
return text
def get_pdf_response(file):
if file is not None:
text = decode_pdf(file)
return get_response(text)
def get_website_response(url):
content = crawl_site(url)
result = get_response(content)
return result
def get_response(text):
print(text)
# split into chunks
text_splitter = CharacterTextSplitter(
separator="\n",
chunk_size=1000,
chunk_overlap=200,
length_function=len
)
chunks = text_splitter.split_text(text)
# create embeddings
embeddings = OpenAIEmbeddings()
knowledge_base = FAISS.from_texts(chunks, embeddings)
return ask_question(knowledge_base)
def ask_question(knowledge_base):
user_question = """this content is a web3 project pitch deck. return result as JSON format. Please use the following JSON format to return data. if some fields are incomplete or missing, use 'N/A' to replace it.
{{"project_name":"this project name","introduction":"project introduction, less than 200 words","slogan":"project slogan","features":"project features","description":"project description","roadmap":"g","fundraising":"fundraising target,round, valuation etc.",contact_email":"project contact email","website":"project official website","twitter":"official twitter","github":"official github","telegram":"official telegram","team_member":"team member list, include name, position, introduction, twitter, github, telegram, etc."}}"""
print("Question:", user_question)
if user_question:
# show user input
docs = knowledge_base.similarity_search(user_question)
llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0.7)
chain = load_qa_chain(llm, chain_type="stuff")
try:
with get_openai_callback() as cb:
response = chain.run(input_documents=docs, question=user_question)
print(f"Total Tokens: {cb.total_tokens}")
print(f"Prompt Tokens: {cb.prompt_tokens}")
print(f"Completion Tokens: {cb.completion_tokens}")
print(f"Total Cost (USD): ${cb.total_cost}")
print("Answer:", response)
json.loads(response)
except json.decoder.JSONDecodeError:
response = {"error": "Data can't found"}
except Timeout:
response = {"error": "Reuest timeout, please try again"}
print(json.dumps(response, ensure_ascii=False))
return response
def upload_file(file):
file_path = file.name
file_size = os.path.getsize(file_path)
print("File size:", file_size)
result = get_pdf_response(file_path)
return result
with gr.Blocks(title="Use AI boost your deal flow - Ventureflow") as demo:
gr.Markdown("# Use AI boost your deal flow")
with gr.Tab("Upload Deck"):
file_input = gr.File(file_types=[".pdf"])
json_output = gr.JSON()
upload_button = gr.UploadButton("Click to Upload a Deck(.pdf))")
upload_button.upload(upload_file, upload_button, json_output)
with gr.Tab("Enter Project website"):
text_input = gr.Textbox(label="Enter Project website")
json_output = gr.JSON()
submit_button = gr.Button("Click to Submit")
submit_button.click(get_website_response, text_input, json_output)
gr.Markdown("""
## Links
- Website: [Ventureflow.xyz](https://ventureflow.xyz)
- Twitter: [@VentureFlow_xyz](https://twitter.com/VentureFlow_xyz)
- App: [app.ventureflow.xyz](https://app.ventureflow.xyz)
- Docs: [docs.ventureflow.xyz](https://docs.ventureflow.xyz)
""")
demo.launch() |