retrieval_pipeline/main.py

from langchain.vectorstores import ElasticVectorSearch
from langchain.llms import OpenAI, HuggingFaceHub
from langchain_community.embeddings import HuggingFaceEmbeddings
from retrieval_pipeline.hybrid_search import HybridRetriever, get_dense_db, get_sparse_retriever
from retrieval_pipeline.utils import get_hybrid_indexes

from langchain.retrievers import ContextualCompressionRetriever
from langchain.retrievers.document_compressors import CrossEncoderReranker
from langchain_community.cross_encoders import HuggingFaceCrossEncoder

import logging
import tqdm


def get_compression_retriever(retriever):
    model = HuggingFaceCrossEncoder(model_name="BAAI/bge-reranker-base")
    compressor = CrossEncoderReranker(model=model, top_n=3)
    compression_retriever = ContextualCompressionRetriever(
        base_compressor=compressor, base_retriever=retriever
    )
    return compression_retriever

# Embedding Models Loader
def get_huggingface_embeddings(model_name):
    logging.info(f"Loading Huggingface Embedding")
    embeddings = HuggingFaceEmbeddings(model_name=model_name)
    return embeddings

def get_vectorstore(index_name, embeddings, elasticsearch_url=None):
    logging.info(f"Loading vectorstore")

    index_dense, index_sparse = get_hybrid_indexes(index_name)

    dense_db = get_dense_db(elasticsearch_url, index_dense, embeddings)
    dense_retriever = dense_db.as_retriever()

    sparse_retriever = get_sparse_retriever(elasticsearch_url, index_sparse)

    hybrid_retriever = HybridRetriever(
        dense_db=dense_db,
        dense_retriever=dense_retriever,
        sparse_retriever=sparse_retriever,
        index_dense=index_dense,
        index_sparse=index_sparse,
        top_k_dense=2,
        top_k_sparse=3
    )

    # db = ElasticVectorSearch(
    #     elasticsearch_url=elasticsearch_url,
    #     index_name=index_name,
    #     embedding=embeddings,
    # )
    return hybrid_retriever

def get_retriever(index, elasticsearch_url):
    # cache.init(pre_embedding_func=get_msg_func)
    # cache.set_openai_key(openai_api_key)

    embeddings = get_huggingface_embeddings(model_name="multilingual-e5-large")

    # llm = get_openai_llm(
    #     model_name=model_name, temperature=0, api_key=model_api_key
    # )
    # embeddings = get_openai_embeddings(embedding_api_key, embedding_name)

    # question_generator = load_question_generator(llm)
    # answer_generator = load_answer_generator(llm, company=model_config['company_name'], tone=model_config['tone'], additional_instructions=model_config['additional_instructions'])

    retriever = get_vectorstore(
        index,
        embeddings=embeddings,
        elasticsearch_url=elasticsearch_url,
    )

    # if history:
    #     qa = get_conversational_chain(retriever, question_generator, answer_generator)
    # else:
    #     qa = get_retrieval_chain(retriever, answer_generator)

    # chain = CustomLLMChain(
    #     chain=qa,
    #     model_name=llm.model_name,
    #     use_history=history
    # )
    # 
    # 
    return retriever

def get_relevant_documents(query, retriever, top_k):
    results = retriever.get_relevant_documents(query)
    passages = [{
        "id": i,
        "text": result.page_content
    } for i, result in enumerate(results)]

    reranked_result = ranker.rerank(RerankRequest(query=query, passages=passages))
    return reranked_result