feat: Update app.py with DataFrameLoader and LongContextReorder

app.py

@@ -2,6 +2,7 @@ import os
 import torch
 
 import gradio as gr
+import pandas as pd
 from dotenv import load_dotenv
 from langchain.callbacks.base import BaseCallbackHandler
 from langchain.embeddings import CacheBackedEmbeddings
@@ -10,7 +11,11 @@ from langchain.retrievers import EnsembleRetriever
 from langchain.storage import LocalFileStore
 from langchain_anthropic import ChatAnthropic
 from langchain_community.chat_models import ChatOllama
-from langchain_community.document_loaders import NotebookLoader, TextLoader
+from langchain_community.document_loaders import (
+    NotebookLoader,
+    TextLoader,
+    DataFrameLoader,
+)
 from langchain_community.document_loaders.generic import GenericLoader
 from langchain_community.document_loaders.parsers.language.language_parser import (
     LanguageParser,
@@ -21,7 +26,11 @@ from langchain_core.callbacks.manager import CallbackManager
 from langchain_core.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 from langchain_core.output_parsers import StrOutputParser
 from langchain_core.prompts import PromptTemplate
-from langchain_core.runnables import ConfigurableField, RunnablePassthrough
+from langchain_core.runnables import (
+    ConfigurableField,
+    RunnablePassthrough,
+    RunnableLambda,
+)
 from langchain_google_genai import GoogleGenerativeAI
 from langchain_groq import ChatGroq
 from langchain_openai import ChatOpenAI, OpenAIEmbeddings
@@ -29,6 +38,8 @@ from langchain_text_splitters import Language, RecursiveCharacterTextSplitter
 
 from langchain_cohere import CohereRerank
 from langchain.retrievers.contextual_compression import ContextualCompressionRetriever
+from langchain_community.document_transformers import LongContextReorder
+
 
 # Load environment variables
 load_dotenv()
@@ -87,10 +98,9 @@ print(f".ipynb 파일의 개수: {len(ipynb_documents)}")
 
 
 ## wikidocs
-import pandas as pd
-
-df = pd.read_parquet("wikidocs_14314.parquet")
-wiki_documents = text_splitter.split_documents(loader.load())
+df = pd.read_parquet("./docs/wikidocs_14314.parquet")
+loader = DataFrameLoader(df, page_content_column="content")
+wiki_documents = loader.load()
 
 
 # Split documents into chunks
@@ -257,7 +267,11 @@ llm = ChatOpenAI(
 
 # Create retrieval-augmented generation chain
 rag_chain = (
-    {"context": compression_retriever, "question": RunnablePassthrough()}
+    {
+        "context": compression_retriever
+        | RunnableLambda(LongContextReorder().transform_documents),
+        "question": RunnablePassthrough(),
+    }
     | prompt
     | llm
     | StrOutputParser()