File size: 3,141 Bytes
a392ebe |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 |
import yaml
import os
import os
from pinecone import Pinecone, ServerlessSpec
from dotenv import load_dotenv
import os
from dotenv import load_dotenv
from llama_index.vector_stores.pinecone import PineconeVectorStore
from llama_index.core import StorageContext, VectorStoreIndex
from llama_index.core import SimpleDirectoryReader
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
from llama_index.core import Settings
from llama_index.llms.groq import Groq
load_dotenv()
script_dir = os.path.dirname(os.path.abspath(__file__))
# Construct the base directory (one level up from the script directory)
base_dir = os.path.dirname(script_dir)
# Construct the path to the config file
config_path = os.path.join(base_dir, 'configs', 'config.yaml')
def load_config(file_path):
with open(file_path, 'r') as file:
config = yaml.safe_load(file)
return config
def index_connection():
# Load the configuration from a YAML file
config = load_config(config_path)
# Initialize the Pinecone client
pc = Pinecone(
api_key=os.getenv('PINECONE_API_KEY')
)
index_name = config['pinecone']['index_name']
index = pc.Index(index_name)
return index
def chunk_documents(directory_path="./data/paul_graham"):
"""
Reads documents from a specified directory and chunks them.
Args:
directory_path (str): The path of the directory containing documents to read.
Returns:
List[Document]: A list of document chunks that will be indexed.
"""
# Load documents from the directory
documents = SimpleDirectoryReader(directory_path).load_data()
# Here you could apply further chunking logic if needed (for example, split large documents into smaller chunks)
# For now, we're assuming the reader does basic chunking for us
return documents
# Part 2: Loading Chunks into Pinecone
def load_chunks_into_pinecone(documents):
config = load_config(config_path)
pinecone_index = index_connection()
model_name = config['model']['model_name']
embed_model_name = config['embeddings']['model_name']
print(embed_model_name)
Settings.llm = Groq(model=model_name, api_key=os.getenv('GROQ_API_KEY'))
Settings.chunk_size = config['pinecone']['dimension']
embed_model = HuggingFaceEmbedding(model_name=embed_model_name)
Settings.embed_model = embed_model
vector_store = PineconeVectorStore(pinecone_index=pinecone_index)
# Create the storage context
storage_context = StorageContext.from_defaults(vector_store=vector_store)
# Create the index with the documents
index = VectorStoreIndex.from_documents(documents, storage_context=storage_context)
print("Data has been successfully loaded into the Pinecone index!")
return index
# Example usage
if __name__ == "__main__":
# Step 1: Chunk the documents
documents = chunk_documents(directory_path=r"C:\Users\agshi\Desktop\Omdena\Canada Policy\TorontoCanadaChapter_CanPolicyInsight\task6_model_deployment\assets")
# Step 2: Load the chunks into Pinecone
index = load_chunks_into_pinecone(documents)
|