import os
from typing import List

from llama_index import (
SimpleDirectoryReader,
Document,
StorageContext,
load_index_from_storage
)

from llama_index.node_parser import SentenceWindowNodeParser
from llama_index.llms import OpenAI
from llama_index.embeddings import OpenAIEmbedding
from llama_index import ServiceContext
from llama_index import VectorStoreIndex
from llama_index.indices.postprocessor import MetadataReplacementPostProcessor
from llama_index.indices.postprocessor import SentenceTransformerRerank


from llama_index.llms import OpenAI

# for loading environment variables
from decouple import config

from trulens_eval import Feedback, Tru, TruLlama
from trulens_eval.feedback import Groundedness
from trulens_eval.feedback.provider.openai import OpenAI as OpenAITruLens

import numpy as np

# set env variables
os.environ[“OPENAI_API_KEY”] = config(“OPENAI_API_KEY”)


# load document
documents = SimpleDirectoryReader(
input_dir=“…/dataFiles/”
).load_data(show_progress=True)


# merge pages into one
document = Document(text=“\n\n”.join([doc.text for doc in documents]))


def create_indexes(
documents: Document,
index_save_dir: str,
window_size: int = 4,
llm_model: str = “gpt-3.5-turbo”,
temperature: float = 0.1
):
node_parser = SentenceWindowNodeParser.from_defaults(
window_size=window_size,
window_metadata_key=“window”,
original_text_metadata_key=“original_text”,
)

# creating OpenAI gpt-3.5-turbo LLM and OpenAIEmbedding model
llm = OpenAI(model=llm_model, temperature=temperature)
embed_model = OpenAIEmbedding()

# creating the service context
sentence_context = ServiceContext.from_defaults(
llm=llm,
embed_model=embed_model,
node_parser=node_parser,
)

if not os.path.exists(index_save_dir):
# creating the vector store index
index = VectorStoreIndex.from_documents(
[document], service_context=sentence_context
)

# make vector store persistant
index.storage_context.persist(persist_dir=index_save_dir)
else:
# load vector store indexed if they exist
index = load_index_from_storage(
StorageContext.from_defaults(persist_dir=index_save_dir),
service_context=sentence_context
)

return index


def create_query_engine(
sentence_index: VectorStoreIndex,
similarity_top_k: int = 6,
rerank_top_n: int = 5,
rerank_model: str = “BAAI/bge-reranker-base”,
):
# add meta data replacement post processor
postproc = MetadataReplacementPostProcessor(
target_metadata_key=“window”
)

# link: https://huggingface.co/BAAI/bge-reranker-base
rerank = SentenceTransformerRerank(
top_n=rerank_top_n,
model=rerank_model
)

sentence_window_engine = sentence_index.as_query_engine(
similarity_top_k=similarity_top_k,
node_postprocessors=[postproc, rerank]
)

return sentence_window_engine


# create index
index = create_indexes(
documents=documents,
index_save_dir=“./sentence_window_size_10_index”,
window_size=3,
llm_model=“gpt-3.5-turbo”,
temperature=0.1
)

# create query engine
sentence_window_engine = create_query_engine(
sentence_index=index,
similarity_top_k=5,
rerank_top_n=2,
)

# RAG pipeline evals
tru = Tru(database_file=“…/default.sqlite”)

openai = OpenAITruLens()

grounded = Groundedness(groundedness_provider=OpenAITruLens())

# Define a groundedness feedback function
f_groundedness = Feedback(grounded.groundedness_measure_with_cot_reasons).on(
TruLlama.select_source_nodes().node.text
).on_output(
).aggregate(grounded.grounded_statements_aggregator)

# Question/answer relevance between overall question and answer.
f_qa_relevance = Feedback(openai.relevance).on_input_output()

# Question/statement relevance between question and each context chunk.
f_qs_relevance = Feedback(openai.qs_relevance).on_input().on(
TruLlama.select_source_nodes().node.text
).aggregate(np.mean)


tru_query_engine_recorder = TruLlama(sentence_window_engine,
app_id=‘sentence_window_size_10’,
feedbacks=[f_groundedness, f_qa_relevance, f_qs_relevance])


eval_questions = []

with open(“./eval_questions.txt”, “r”) as eval_qn:
for qn in eval_qn:
qn_stripped = qn.strip()
eval_questions.append(qn_stripped)


def run_eval(eval_questions: List[str]):
for qn in eval_questions:
# eval using context window
with tru_query_engine_recorder as recording:
sentence_window_engine.query(qn)


run_eval(eval_questions=eval_questions)

# run dashboard
tru.run_dashboard()


from typing import List
from llama_index import (
Document,
VectorStoreIndex,
SimpleDirectoryReader,
ServiceContext,
)
from llama_index.retrievers import RecursiveRetriever
from llama_index.query_engine import RetrieverQueryEngine
from llama_index.node_parser import SimpleNodeParser
from llama_index.embeddings import OpenAIEmbedding
from llama_index.schema import IndexNode
from llama_index.llms import OpenAI

# for loading environment variables
from decouple import config

import os

from trulens_eval import Feedback, Tru, TruLlama
from trulens_eval.feedback import Groundedness
from trulens_eval.feedback.provider.openai import OpenAI as OpenAITruLens

import numpy as np

# set env variables
os.environ[“OPENAI_API_KEY”] = config(“OPENAI_API_KEY”)

# create LLM and Embedding Model
embed_model = OpenAIEmbedding()
llm = OpenAI(model=“gpt-3.5-turbo”)
service_context = ServiceContext.from_defaults(
embed_model=embed_model, llm=llm)

# load data
documents = SimpleDirectoryReader(
input_dir=“…/dataFiles”).load_data(show_progress=True)

doc_text = “\n\n”.join([d.get_content() for d in documents])
docs = [Document(text=doc_text)]

# create nodes parser
node_parser = SimpleNodeParser.from_defaults(chunk_size=1024)

# split into nodes
base_nodes = node_parser.get_nodes_from_documents(documents=docs)

# set document IDs
for idx, node in enumerate(base_nodes):
node.id_ = f"node-{idx}“

# create parent child documents
sub_chunk_sizes = [128, 256, 512]
sub_node_parsers = [
SimpleNodeParser.from_defaults(chunk_size=c, chunk_overlap=0) for c in sub_chunk_sizes
]

all_nodes = []
for base_node in base_nodes:
for n in sub_node_parsers:
sub_nodes = n.get_nodes_from_documents([base_node])
sub_inodes = [
IndexNode.from_text_node(sn, base_node.node_id) for sn in sub_nodes
]
all_nodes.extend(sub_inodes)

# also add original node to node
original_node = IndexNode.from_text_node(base_node, base_node.node_id)
all_nodes.append(original_node)


all_nodes_dict = {n.node_id: n for n in all_nodes}

# creating index
index = VectorStoreIndex(nodes=all_nodes, service_context=service_context)

# creating a chunk retriever
vector_retriever_chunk = index.as_retriever(similarity_top_k=2)


retriever_chunk = RecursiveRetriever(
“vector”,
retriever_dict={“vector”: vector_retriever_chunk},
node_dict=all_nodes_dict,
verbose=True,
)

query_engine_chunk = RetrieverQueryEngine.from_args(
retriever_chunk, service_context=service_context
)

# RAG pipeline evals
tru = Tru(database_file=”…/default.sqlite")

openai = OpenAITruLens()

grounded = Groundedness(groundedness_provider=OpenAITruLens())

# Define a groundedness feedback function
f_groundedness = Feedback(grounded.groundedness_measure_with_cot_reasons).on(
TruLlama.select_source_nodes().node.text
).on_output(
).aggregate(grounded.grounded_statements_aggregator)

# Question/answer relevance between overall question and answer.
f_qa_relevance = Feedback(openai.relevance).on_input_output()

# Question/statement relevance between question and each context chunk.
f_qs_relevance = Feedback(openai.qs_relevance).on_input().on(
TruLlama.select_source_nodes().node.text
).aggregate(np.mean)


tru_query_engine_recorder = TruLlama(query_engine_chunk,
app_id=‘Parent_document_retrieval’,
feedbacks=[f_groundedness, f_qa_relevance, f_qs_relevance])

eval_questions = []

with open(“./eval_questions.txt”, “r”) as eval_qn:
for qn in eval_qn:
qn_stripped = qn.strip()
eval_questions.append(qn_stripped)


def run_eval(eval_questions: List[str]):
for qn in eval_questions:
# eval using context window
with tru_query_engine_recorder as recording:
query_engine_chunk.query(qn)


run_eval(eval_questions=eval_questions)

# run dashboard
tru.run_dashboard()


from typing import List
from llama_index import (
StorageContext,
VectorStoreIndex,
SimpleDirectoryReader,
ServiceContext,
load_index_from_storage
)
from llama_index.node_parser import SimpleNodeParser
from llama_index.embeddings import OpenAIEmbedding
from llama_index.llms import OpenAI
from llama_index.query_engine import RetrieverQueryEngine

from trulens_eval import Feedback, Tru, TruLlama
from trulens_eval.feedback import Groundedness
from trulens_eval.feedback.provider.openai import OpenAI as OpenAITruLens

import numpy as np

# for loading environment variables
from decouple import config

import os

# set env variables
os.environ[“OPENAI_API_KEY”] = config(“OPENAI_API_KEY”)


# create LLM and Embedding Model
embed_model = OpenAIEmbedding()
llm = OpenAI(model=“gpt-3.5-turbo”)
service_context = ServiceContext.from_defaults(
embed_model=embed_model, llm=llm)


# check if data indexes already exists
if not os.path.exists(“./storage”):
# load data
documents = SimpleDirectoryReader(
input_dir=“…/dataFiles”).load_data(show_progress=True)

# create nodes parser
node_parser = SimpleNodeParser.from_defaults(chunk_size=1024)

# split into nodes
base_nodes = node_parser.get_nodes_from_documents(documents=documents)

# creating index
index = VectorStoreIndex(nodes=base_nodes, service_context=service_context)

# store index
index.storage_context.persist()
else:
# load existing index
storage_context = StorageContext.from_defaults(persist_dir=“./storage”)
index = load_index_from_storage(storage_context=storage_context)


# create retriever
retriever = index.as_retriever(similarity_top_k=2)

# query retriever engine
query_engine = RetrieverQueryEngine.from_args(
retriever=retriever,
service_context=service_context
)

# RAG pipeline evals
tru = Tru(database_file=“…/default.sqlite”)

openai = OpenAITruLens()

grounded = Groundedness(groundedness_provider=OpenAITruLens())

# Define a groundedness feedback function
f_groundedness = Feedback(grounded.groundedness_measure_with_cot_reasons).on(
TruLlama.select_source_nodes().node.text
).on_output(
).aggregate(grounded.grounded_statements_aggregator)

# Question/answer relevance between overall question and answer.
f_qa_relevance = Feedback(openai.relevance).on_input_output()

# Question/statement relevance between question and each context chunk.
f_qs_relevance = Feedback(openai.qs_relevance).on_input().on(
TruLlama.select_source_nodes().node.text
).aggregate(np.mean)


tru_query_engine_recorder = TruLlama(query_engine,
app_id=‘Basic_RAG’,
feedbacks=[f_groundedness, f_qa_relevance, f_qs_relevance])

eval_questions = []

with open(“./eval_questions.txt”, “r”) as eval_qn:
for qn in eval_qn:
qn_stripped = qn.strip()
eval_questions.append(qn_stripped)


def run_eval(eval_questions: List[str]):
for qn in eval_questions:
# eval using context window
with tru_query_engine_recorder as recording:
query_engine.query(qn)


run_eval(eval_questions=eval_questions)

# run dashboard
tru.run_dashboard()


can you combine above three code into one streamlit app using advance oops technique, use streamlit tab to switch between code