# Hog RAGger

## Project Overview
Hog RAGger is a Retrieval-Augmented Generation (RAG) pipeline designed to answer complex queries using a corpus of web-scraped articles. The project aims to develop an advanced RAG system capable of handling queries that require evidence from multiple documents and incorporate document metadata.

## Dataset
The project uses the Hog RAGger Dataset, which consists of two main components:

1. **Corpus.json**: A collection of web-scraped articles containing:
   - Title
   - Author
   - Source
   - Publishing date
   - Category
   - Article URL
   - Article body (text content)

2. **Train.json**: Contains 2556 queries processed by a state-of-the-art RAG pipeline, including:
   - Queries requiring evidence from 2-4 documents
   - Incorporation of document metadata
   - Complex scenarios mimicking real-world RAG applications

Dataset link: [Hog RAGger Dataset](insert_actual_link_here)

## Project Goal
The main objective is to develop a RAG pipeline that can:
1. Answer user queries based on the provided corpus
2. Utilize evidence from 2-4 documents per query
3. Incorporate document metadata when necessary
4. Format output to match the structure in train.json

## Output Format
The pipeline should produce output in the following format:

```json
{
  "query": "User's question",
  "answer": "Generated answer based on evidence",
  "question_type": "inference_query",
  "evidence_list": [
    {
      "title": "Article title",
      "author": "Author name",
      "url": "Article URL",
      "source": "Publication source",
      "category": "Article category",
      "published_at": "Publication date and time",
      "fact": "Extracted fact supporting the answer"
    },
    // Additional evidence items...
  ]
}
```

## Getting Started
(Include instructions for setting up the project, installing dependencies, and running the pipeline)

## Usage
(Provide examples of how to use the RAG pipeline, including sample input and output)

## Evaluation
(Describe how the performance of the RAG pipeline will be evaluated, potentially using metrics like accuracy, relevance, and completeness of answers)

## Contributing
(Include guidelines for contributing to the project, if applicable)

COmplete this readame file based on this code 
import streamlit as st
import json
from typing import List, Dict, Any
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM

# Load corpus and create embeddings
@st.cache_resource
def load_corpus(file_path: str) -> List[Dict[str, Any]]:
    with open(file_path, 'r') as f:
        return json.load(f)

@st.cache_data
def preprocess_corpus(corpus: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    processed_corpus = []
    for doc in corpus:
        processed_doc = {
            'title': doc['title'],
            'author': doc['author'],
            'url': doc['url'],
            'source': doc['source'],
            'category': doc['category'],
            'published_at': doc['published_at'],
            'content': doc['title'] + ' ' + doc['body']
        }
        processed_corpus.append(processed_doc)
    return processed_corpus

@st.cache_resource
def create_embeddings(corpus: List[Dict[str, Any]], model_name: str = 'sentence-transformers/all-MiniLM-L6-v2') -> np.ndarray:
    model = SentenceTransformer(model_name)
    return model.encode([doc['content'] for doc in corpus])

def retrieve_documents(query: str, corpus: List[Dict[str, Any]], embeddings: np.ndarray, model_name: str = 'sentence-transformers/all-MiniLM-L6-v2', top_k: int = 4) -> List[Dict[str, Any]]:
    model = SentenceTransformer(model_name)
    query_embedding = model.encode([query])
    similarities = cosine_similarity(query_embedding, embeddings)[0]
    top_indices = np.argsort(similarities)[-top_k:][::-1]
    return [corpus[i] for i in top_indices]

@st.cache_resource
def load_qa_model(model_name: str = 'google/flan-t5-base'):
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
    return tokenizer, model

def generate_answer(query: str, relevant_docs: List[Dict[str, Any]], tokenizer, model) -> str:
    context = " ".join([doc['content'] for doc in relevant_docs])
    input_text = f"Query: {query}\nContext: {context}\nAnswer:"
    
    inputs = tokenizer(input_text, return_tensors="pt", max_length=1024, truncation=True)
    outputs = model.generate(**inputs, max_length=150, num_return_sequences=1, temperature=0.7)
    
    return tokenizer.decode(outputs[0], skip_special_tokens=True)

def rag_pipeline(query: str, corpus: List[Dict[str, Any]], embeddings: np.ndarray, tokenizer, model) -> Dict[str, Any]:
    relevant_docs = retrieve_documents(query, corpus, embeddings)
    answer = generate_answer(query, relevant_docs, tokenizer, model)
    
    evidence_list = []
    for doc in relevant_docs:
        evidence = {
            'title': doc['title'],
            'author': doc['author'],
            'url': doc['url'],
            'source': doc['source'],
            'category': doc['category'],
            'published_at': doc['published_at'],
            'fact': doc['content'][:200] + '...'
        }
        evidence_list.append(evidence)
    
    return {
        'query': query,
        'answer': answer,
        'question_type': 'inference_query',
        'evidence_list': evidence_list
    }

# Streamlit app
st.set_page_config(page_title="RAG Query Processor", page_icon="🔍", layout="wide")

st.title("RAG Query Processor")

# Load data and models
corpus = load_corpus('corpus.json')
processed_corpus = preprocess_corpus(corpus)
embeddings = create_embeddings(processed_corpus)
tokenizer, model = load_qa_model()

# Query input
query = st.text_input("Enter your query:", "What is the impact of COVID-19 on the economy?")

if st.button("Process Query"):
    with st.spinner("Processing query..."):
        result = rag_pipeline(query, processed_corpus, embeddings, tokenizer, model)
    
    # Display results
    st.subheader("Answer")
    st.write(result['answer'])
    
    st.subheader("Evidence")
    for evidence in result['evidence_list']:
        with st.expander(evidence['title']):
            st.write(f"**Author:** {evidence['author']}")
            st.write(f"**Source:** {evidence['source']}")
            st.write(f"**Category:** {evidence['category']}")
            st.write(f"**Published:** {evidence['published_at']}")
            st.write(f"**Excerpt:** {evidence['fact']}")
            st.write(f"[Read More]({evidence['url']})")

st.sidebar.title("About")
st.sidebar.info("This app uses a RAG (Retrieval-Augmented Generation) pipeline to answer queries based on a corpus of documents.")