main.py

import os
from dotenv import load_dotenv
from langchain_community.document_loaders import PyPDFLoader
from langchain_community.llms import OpenAI
from langchain.chains.summarize import load_summarize_chain
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain import hub
from langchain_community.vectorstores import Qdrant
from qdrant_client import QdrantClient
from langchain_openai import OpenAIEmbeddings
from langchain_openai import ChatOpenAI
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnablePassthrough
from langchain.prompts import PromptTemplate

load_dotenv()

def summarize_pdf_document(file_path, openai_api_key):
    # Load PDF document
    loader = PyPDFLoader(file_path)
    documents = loader.load()

    # Split text from documents
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=500)
    docs = text_splitter.split_documents(documents)

    # Set up OpenAI client
    llm = OpenAI(temperature=0, openai_api_key=openai_api_key)

    # Load the summarization chain process
    summary_chain = load_summarize_chain(llm=llm, chain_type='map_reduce')

    # Invoke the summarization process
    output = summary_chain.run(docs)

    return output

def embed_document_data(documents):
    """Load, process, and embed a PDF file into a vector store.

    Args:
        file_path (str): Path to the PDF file to be processed and embedded.
    """


    # Split text from documents into smaller chunks
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=400)
    texts = text_splitter.split_documents(documents)

    # Set up embeddings model with OpenAI
    openai_api_key = os.getenv("OPENAI_API_KEY")
    embeddings_model = OpenAIEmbeddings(model='text-embedding-3-small', openai_api_key=openai_api_key)

    # Configure Qdrant client
    qdrant_url = os.getenv("QDRANT_URL")
    qdrant_api_key = os.getenv("QDRANT_API_KEY")
    client = QdrantClient(location=qdrant_url, api_key=qdrant_api_key)

    # Initialize Qdrant storage with the client and embedding model
    qdrant = Qdrant(client=client, collection_name="Lex-v1", embeddings=embeddings_model)

    # Add documents to the Qdrant collection
    qdrant.add_documents(texts)


def retrieve_documents(query: str):
    """
    Takes a user query as input and returns a response using a Retrieval-Augmented Generation (RAG) flow
    incorporating langchain, Qdrant, and OpenAI.

    Args:
        query (str): The user's question to be answered.

    """
    try:
        # Setup
        qdrant_url = os.getenv('QDRANT_URL')
        qdrant_api_key = os.getenv("QDRANT_API_KEY")
        openai_api_key=os.getenv('OPENAI_API_KEY')

        embeddings_model = OpenAIEmbeddings(model='text-embedding-3-small', openai_api_key=openai_api_key)
        qdrant_client = QdrantClient(url=qdrant_url, api_key=qdrant_api_key)
        qdrant = Qdrant(client=qdrant_client, collection_name="Lex-v1",
                        embeddings=embeddings_model)
        retriever = qdrant.as_retriever(search_kwargs={"k": 5})


        prompt=PromptTemplate(

       template="""
        # Your role
        You are a brilliant expert at understanding the intent of the questioner and the crux of the question, and providing the most optimal answer  from the docs to the questioner's needs from the documents you are given.


        # Instruction
        Your task is to answer the question  using the following pieces of retrieved context delimited by XML tags.

       <retrieved context>
        Retrieved Context:
        {context}
        </retrieved context>


        # Constraint
        1. Think deeply and multiple times about the user's question\nUser's question:\n{question}\nYou must understand the intent of their question and provide the most appropriate answer.
        - Ask yourself why to understand the context of the question and why the questioner asked it, reflect on it, and provide an appropriate response based on what you understand.
        2. Choose the most relevant content(the key content that directly relates to the question) from the retrieved context and use it to generate an answer.
        3. Generate a concise, logical answer. When generating the answer, Do Not just list your selections, But rearrange them in context so that they become paragraphs with a natural flow.
        4. When you don't have retrieved context for the question or If you have a retrieved documents, but their content is irrelevant to the question, you should answer 'I can't find the answer to that question in the material I have'.
        5. Use five sentences maximum. Keep the answer concise but logical/natural/in-depth.
        6. At the end of the response provide metadata provided in the relevant docs , For example:"Metadata: page: 19, source: /content/OCR_RSCA/Analyse docs JVB + mails et convention FOOT INNOVATION.pdf'.Return just the page and source.Provide a list of all the metadata found in the Relevent content formatted as bullets



        # Question:
        {question}""",
        input_variables=["context","question"]
        )
        llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0,openai_api_key=openai_api_key)


        def format_docs(docs):
          formatted_docs = []
          for doc in docs:
           # Format the metadata into a string
            metadata_str = ', '.join(f"{key}: {value}" for key, value in doc.metadata.items())

          # Combine page content with its metadata
            doc_str = f"{doc.page_content}\nMetadata: {metadata_str}"

          # Append to the list of formatted documents
            formatted_docs.append(doc_str)

          # Join all formatted documents with double newlines
          return "\n\n".join(formatted_docs)


        rag_chain = (
        {"context": retriever | format_docs, "question": RunnablePassthrough()}
        | prompt
        | llm
        | StrOutputParser()
      )

        result = rag_chain.invoke(query)
        return result
    except Exception as e:
      print(f"Error processing the query: {e}")
      return None
    
def is_document_embedded(filename):
    """Check if a document has already been embedded based on its filename."""
    # This function needs to query your backend or check a local database/file.
    # For simplicity, here's a placeholder that always returns False.
    # Replace this with actual logic.
    return False