File size: 7,733 Bytes
52e3677
 
 
 
 
 
 
 
 
 
 
6a6f2fa
52e3677
 
 
 
 
 
6a6f2fa
 
52e3677
 
 
 
 
 
 
 
 
 
 
 
 
 
6a6f2fa
 
 
 
 
 
 
 
 
 
 
 
52e3677
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6a6f2fa
52e3677
 
6a6f2fa
52e3677
 
 
 
 
 
 
6a6f2fa
 
52e3677
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6a6f2fa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
52e3677
 
6a6f2fa
 
52e3677
 
 
 
 
 
 
 
 
 
 
7123d7a
52e3677
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6a6f2fa
 
 
 
52e3677
6a6f2fa
52e3677
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6a6f2fa
 
52e3677
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
import json
from dotenv import load_dotenv
import gradio as gr
import os
from PyPDF2 import PdfReader
from langchain.text_splitter import CharacterTextSplitter
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.vectorstores import FAISS
from langchain.chains.question_answering import load_qa_chain
from langchain.chat_models import ChatOpenAI
from langchain.callbacks import get_openai_callback
# from requests.exceptions import Timeout
import requests
from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin
import time
import random
import os
import mimetypes
from openai.error import Timeout

OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")

load_dotenv()


knowledge_base = None


headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
}


def is_webpage(url):
    """
    判断一个链接是否为网页链接
    """
    content_type = requests.head(url, headers=headers).headers.get('Content-Type')
    if content_type is not None:
        mimetype, encoding = mimetypes.guess_type(url, strict=False)
        if mimetype is not None and mimetype.startswith('text/html'):
            return True
    return False


def get_internal_links(url):
    print('start get internal links')
    internal_links = []
    domain = urlparse(url).netloc  # 获取当前网站域名
    response = requests.get(url, headers=headers, timeout=5)
    soup = BeautifulSoup(response.content, 'html.parser')
    for a in soup.find_all('a', href=True):
        href = a['href']
        if href.startswith('http'):  # 外链
            if urlparse(href).netloc == domain:  # 如果是本站链接
                internal_links.append(href)
        else:  # 内链
            internal_link = urljoin(url, href)
            if urlparse(internal_link).netloc == domain:
                internal_links.append(internal_link)
    internal_links = list(set(internal_links))
    print(internal_links)
    return internal_links




def get_page_content(url):
    
    response = requests.get(url, headers=headers, timeout=5)
    soup = BeautifulSoup(response.content, 'html.parser')
    content = soup.get_text('\n')


    time.sleep(random.randint(1, 3))
    return content

def crawl_site(url):

    # links_to_visit = get_internal_links(url)
    links_to_visit = [url]

    content = ""
    
    while links_to_visit:
        link = links_to_visit.pop(0)
    
        content += get_page_content(link)
        print(f'Page content for {link}:\n')
    return content


def decode_pdf(file_path):
    encodings = ['utf-8', 'gbk', 'gb2312', 'big5', 'cp1252']  # 常见编码方式
    text = ""
    with open(file_path, 'rb') as f:
        pdf_reader = PdfReader(f)
        for encoding in encodings:
            try:
                for page in pdf_reader.pages:
                    temp_text = page.extract_text()
                    encode_temp_text = temp_text.encode(encoding)
                    decode_temp_text = encode_temp_text.decode(encoding,'strict')
                    text += decode_temp_text
                break
            except UnicodeDecodeError:
                pass
    return text


def get_pdf_response(file):
    if file is not None:
        text = decode_pdf(file)

        print('pdf text:', text)

        if text:
            return get_response(text)
        else:
            return {"error": "covert pdf to text failed"}
        
def fix_url(url):
    try:
        response = requests.head(url)
        if response.status_code != 405:
            return url
        else:
            return "https://" + url
    except requests.exceptions.MissingSchema:
        return "https://" + url


def get_website_response(url):

    url = fix_url(url)
    content = crawl_site(url)
    result = get_response(content)

    return result


def get_response(text):

        # split into chunks
        text_splitter = CharacterTextSplitter(
            separator="\n",
            chunk_size=1000,
            chunk_overlap=200,
            length_function=len
        )
        chunks = text_splitter.split_text(text)
        
        # create embeddings
        embeddings = OpenAIEmbeddings()

   
        knowledge_base = FAISS.from_texts(chunks, embeddings)

        return ask_question(knowledge_base)


def ask_question(knowledge_base):


#     user_question = """this content is a web3 project pitch deck. return result as JSON format. Please use the following JSON format to return data. if some fields are incomplete or missing, use 'N/A' to replace it.
# {{"project_name":"this project name","introduction":"project introduction, less than 200 words","slogan":"project slogan","features":"project features","description":"project description","roadmap":"g","fundraising":"fundraising target,round, valuation etc."}}"""


    user_question = """this content is a web3 project pitch deck. return result as JSON format. Please use the following JSON format to return data. if some fields are incomplete or missing, use 'N/A' to replace it.
    {{"project_name":"this project name","introduction":"project introduction, less than 200 words","slogan":"project slogan","features":"project features","description":"project description","roadmap":"g","fundraising":"fundraising target,round, valuation etc.",contact_email":"project contact email","website":"project official website","twitter":"official twitter","github":"official github","telegram":"official telegram"}}"""

    print("Question:", user_question)
    
    
    if user_question:
        # show user input
        docs = knowledge_base.similarity_search(user_question)

        llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0.7)
        chain = load_qa_chain(llm, chain_type="stuff")

        try:
            with get_openai_callback() as cb:
                response = chain.run(input_documents=docs, question=user_question)
                print(f"Total Tokens: {cb.total_tokens}")
                print(f"Prompt Tokens: {cb.prompt_tokens}")
                print(f"Completion Tokens: {cb.completion_tokens}")
                print(f"Total Cost (USD): ${cb.total_cost}")
            
                print("Answer:", response)

            json.loads(response)
        except json.decoder.JSONDecodeError:
           response = {"error": "Data can't found"}

        except Timeout:
            response = {"error": "Reuest timeout, please try again"}

        print(json.dumps(response, ensure_ascii=False))
        return response
    
    


def upload_file(file):
    file_path = file.name
    file_size = os.path.getsize(file_path)
    print("File size:", file_size)

    result = get_pdf_response(file_path)

    return result




with gr.Blocks(title="Use AI boost your deal flow - Ventureflow") as demo:
    gr.Markdown("# Use AI boost your deal flow")
    with gr.Tab("Upload Deck"):
        # file_input = gr.File(file_types=[".pdf"])
        upload_button = gr.UploadButton("Click to Upload a Deck(.pdf))", file_types=[".pdf"])
        json_output = gr.JSON()
        upload_button.upload(upload_file, upload_button, json_output)
    with gr.Tab("Enter Project website"):
        text_input = gr.Textbox(label="Enter Project website")
        json_output = gr.JSON()
        submit_button = gr.Button("Click to Submit")
        submit_button.click(get_website_response, text_input, json_output)
    gr.Markdown("""
    ## Links
    - Website: [Ventureflow.xyz](https://ventureflow.xyz)
    - Twitter: [@VentureFlow_xyz](https://twitter.com/VentureFlow_xyz)
    - App: [app.ventureflow.xyz](https://app.ventureflow.xyz)
    - Docs: [docs.ventureflow.xyz](https://docs.ventureflow.xyz)
    """)

demo.launch()