Upload localKB_construct.py

localKB_construct.py

@@ -2,10 +2,12 @@
 1.更新了llama-index的库。对应的函数名和用法都有所改变。
 '''
 
+# import gradio as gr
 import openai
 import requests
 import csv
 from llama_index import PromptHelper
+# from llama_index import GPTSimpleVectorIndex ## renamed in the latest version. 
 from llama_index import LLMPredictor
 from llama_index import ServiceContext
 from langchain.chat_models import ChatOpenAI
@@ -18,7 +20,6 @@ import math
 import pandas as pd
 import numpy as np
 import PyPDF2
-# from llama_index import SimpleDirectoryReader, GPTListIndex, readers, GPTSimpleVectorIndex, LLMPredictor, PromptHelper #* working in the previous version. 
 
 ##* in the latest version: GPTSimpleVectorIndex was renamed to GPTVectorStoreIndex, try removing it from the end of your imports
 from llama_index import SimpleDirectoryReader, GPTListIndex, readers, GPTVectorStoreIndex, LLMPredictor, PromptHelper
@@ -28,7 +29,7 @@ from llama_index import download_loader
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 import sys
 import os
-
+from rich import print
 
 ## enironment settings. 
 os.environ["OPENAI_API_KEY"] = "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"
@@ -72,27 +73,36 @@ def construct_index(directory_path):
     
     ## 如果是PDF文件，那么需要用如下命令。注意与txt文件的区别。切需要from llama_index import download_loader。 
     #NOTE: 这里可以问：give me an example of GPT-4 solving math problem. 会回答关于这个PDF中的内容，所以可以确认这个程序调用了in-context learning的功能。
-    CJKPDFReader = download_loader("CJKPDFReader")
-    loader = CJKPDFReader()
+    # CJKPDFReader = download_loader("CJKPDFReader") ## 最新的版本好像不行了，需要用下面的命令。
+    # loader = CJKPDFReader()
+    PDFReader = download_loader("PDFReader") # working。
+    loader = PDFReader()
     # documents = loader.load_data(file=directory_path) #! 注意这里是指向文件本身，而不同于txt文件的指文件夹。
+    print('directory_path now:', directory_path)
+    # print('111')
+    # documents = loader.load_data(file="/Users/yunshi/Downloads/txt_dir/Sparks_of_AGI.pdf") #! 注意这里是指向文件本身，而不同于txt文件的指文件夹。
     documents = loader.load_data(file=directory_path) #! 注意这里是指向文件本身，而不同于txt文件的指文件夹。
+    print('222')
     # index = GPTSimpleVectorIndex(
     #     documents, llm_predictor=llm_predictor, prompt_helper=prompt_helper
     # )
 
     # index = GPTSimpleVectorIndex.from_documents(documents, service_context=service_context) ## oringinal version, working. 
+    # print('documents:', documents)
     index = GPTVectorStoreIndex.from_documents(documents, service_context=service_context) #* the funciton renamed. 
+    print('333')
+
     # index.save_to_disk('/Users/yunshi/Downloads/txt_dir/index.json') ## in the latest version, this function is not working.
 
     return index, service_context
 
-def process_file():
+def process_file(file_path,username):
     print('process_file starts')
-    file_path = "/Users/yunshi/Downloads/txt_dir/Sparks_of_AGI.pdf"
+    # file_path = "/Users/yunshi/Downloads/txt_dir/Sparks_of_AGI.pdf"
     #! 第一次运行是需要开启这个function。如果测试通过index，因此不需要在运行了。记得上传PDF和JSON文件到云服务器上。
     index, service_context = construct_index(file_path)
     # index.storage_context.persist(persist_dir="/Users/yunshi/Downloads/txt_dir/") #* 存储到本地，为以后调用。
-    index.storage_context.persist(persist_dir=f"./") #* 存储到本地，为以后调用。
+    index.storage_context.persist(persist_dir=f"./{username}/") #* 存储到本地，为以后调用。
     print(index)
 
-process_file()
+# process_file(file_path)