Spaces:
Sleeping
Sleeping
File size: 2,492 Bytes
5c50619 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
from llama_index import SimpleDirectoryReader, VectorStoreIndex, LLMPredictor, PromptHelper, ServiceContext,StorageContext
from langchain import OpenAI
import logging
import sys
import os
import json
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
def updateBrain(brainName, files):
print(len(files))
print("Creating Temp brain")
print(brainName)
# set maximum input size
documents = []
corrupt = []
newfiles = []
i = 0
for file in files:
loader = SimpleDirectoryReader(input_files=[file.name])
try:
document1 = loader.load_data()
for doc in document1:
documents.append(doc)
newfiles.append(file.name)
except Exception as e:
print(e)
if (str(e) == "Cannot read an empty file"):
return "Please Wait! Files are uploading, Try again Later!"
corrupt.append(os.path.basename(file.name))
i = i+1
print(i)
storage_context = StorageContext.from_defaults()
service_context = ServiceContext.from_defaults(chunk_size=600,chunk_overlap=10)
index = VectorStoreIndex.from_documents([], storage_context=storage_context,service_context=service_context)
j = 1
for doc in documents:
print(newfiles[j-1])
try:
index.insert(doc)
except Exception as e:
corrupt.append(os.path.basename(newfiles[j-1]))
print("ERROR : "+str(e))
print(j)
j = j+1
# path = "data/"+brainName+".json"
# a = index.save_to_dict()
# with open(path, "w") as f:
# json.dump(a, f, indent=2)
index_name="./"+brainName
index.storage_context.persist(persist_dir=index_name)
print(brainName+" Temp Brain Created!")
for filename in os.listdir("data"):
file_path = os.path.join("data", filename)
print(file_path)
print(corrupt)
current_dir = os.getcwd()
# List all directories in the current directory
folders = [name for name in os.listdir(current_dir) if os.path.isdir(os.path.join(current_dir, name))]
# Print the list of folders
for folder in folders:
print(folder)
if (len(corrupt) > 0):
return """Brain Created!
Below files are corrupt/unformatted, and not added to the brain.
""" + str(corrupt)
return brainName+" Temp Brain Created!"
|