import os import streamlit as st from llama_index import GPTSimpleVectorIndex, SimpleDirectoryReader, ServiceContext,LLMPredictor from langchain.chat_models import ChatOpenAI from llama_index.llm_predictor.chatgpt import ChatGPTLLMPredictor import huggingface_hub from huggingface_hub import Repository from datetime import datetime import csv DATASET_REPO_URL = "https://huggingface.co/datasets/diazcalvi/kionlinde"#"https://huggingface.co/datasets/julien-c/persistent-space-dataset" DATA_FILENAME = "kion.json" DATA_FILE = os.path.join("data", DATA_FILENAME) HF_TOKEN = os.environ.get("HF_TOKEN") print("is none?", HF_TOKEN is None) print("hfh", huggingface_hub.__version__) #os.system("git config --global user.name \"Carlos Diaz\"") #os.system("git config --global user.email \"diazcalvi@gmail.com\"") ##repo = Repository( # local_dir="data", clone_from=DATASET_REPO_URL, use_auth_token=HF_TOKEN #) index_name = "./data/kion.json" documents_folder = "./documents" #@st.experimental_memo #@st.cache_resource def initialize_index(index_name, documents_folder): #llm_predictor = ChatGPTLLMPredictor() llm_predictor = LLMPredictor(llm=ChatOpenAI(temperature=0, model_name="gpt-3.5-turbo")) # text-davinci-003"))"gpt-3.5-turbo" service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor) if os.path.exists(index_name): index = GPTSimpleVectorIndex.load_from_disk(index_name) else: documents = SimpleDirectoryReader(documents_folder).load_data() index = GPTSimpleVectorIndex.from_documents(documents) index.save_to_disk(index_name) print(DATA_FILE) index.save_to_disk(DATA_FILE) return index #@st.experimental_memo #@st.cache_data(max_entries=200, persist=True) def query_index(_index, query_text): response = _index.query(query_text) return str(response) def generate_html() -> str: with open(DATA_FILE) as csvfile: reader = csv.DictReader(csvfile) rows = [] for row in reader: rows.append(row) rows.reverse() if len(rows) == 0: return "no messages yet" else: html = "