app.py

import gradio as gr
import pandas as pd
import evaluate
import theme

default_css = """\
<style type="text/css">
    .diff {
        border: 1px solid #cccccc;
        background: none repeat scroll 0 0 #f8f8f8;
        font-family: 'Bitstream Vera Sans Mono','Courier',monospace;
        font-size: 12px;
        line-height: 1.4;
        white-space: normal;
        word-wrap: break-word;
    }
    .diff div:hover {
        background-color:#ffc;
    }
    .diff .control {
        background-color: #eaf2f5;
        color: #999999;
    }
    .diff .insert {
        background-color: #ddffdd;
        color: #000000;
    }
    .diff .insert .highlight {
        background-color: #aaffaa;
        color: #000000;
    }
    .diff .delete {
        background-color: #ffdddd;
        color: #000000;
    }
    .diff .delete .highlight {
        background-color: #ffaaaa;
        color: #000000;
    }
</style>
"""


df = pd.read_csv("./wiki_bio_gpt3_hallucination.csv")

title = "<h1 style='text-align: center; color: #333333; font-size: 40px;'> Maya, Mithya and AI Hallucinations </h1>"

description = "AI hallucinations can be seen as manifestations of maya or mithya within artificial intelligence systems. Just as maya obscures the true nature of reality, AI hallucinations distort the accurate representation of reality within the realm of AI. These hallucinations occur when AI systems generate outputs that diverge from objective reality or are inconsistent with the data they have been trained on."

titleRAG = "<h1 style='text-align: center; color: #333333; font-size: 40px;'> Routing , Chaining and Branching of RAG documents </h1>"
descriptionRag = " This novel approach involves creating a router across multiple Rag documents, chaining them together, and incorporating branching for greater flexibility and adaptability."
style = theme.Style()

import numpy as np
import pandas as pd
import ast
from openai import OpenAI 
import os
import torch
import spacy
from selfcheckgpt.modeling_selfcheck import SelfCheckMQAG, SelfCheckBERTScore
import en_core_web_sm
nlp = en_core_web_sm.load()


os.environ["OPENAI_API_KEY"]=os.environ['API_TOKEN']
client = OpenAI()
torch.manual_seed(28)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(device)

selfcheck_mqag = SelfCheckMQAG(device=device)
selfcheck_bertscore = SelfCheckBERTScore()
#nlp = spacy.load("en_core_web_sm")

from selfcheckgpt.modeling_mqag import MQAG
mqag_model = MQAG(
    g1_model_type='race', # race (more abstractive), squad (more extractive)
    device=device
)

document =r"""This is the complete recipe cooking paneer butter masala.Heat 1 teaspoon of oil in a pan on medium heat. Once the oil is hot, add the bay leaf, cinnamon stick, cloves and saute for few seconds.Then add the the onion, garlic, ginger and saute for 2 to 3 minutes until the onion is translucent. Add the tomatoes and cashews and mix. Then add 1 cup of water.Cover the pan and cook on medium heat for 15 minutes.After 15 minutes, remove the pan from heat. Remove the bay leaf, cinnamon stick and cloves.Let the mixture cool down a bit and then transfer to a blender. It’s important to let it cool down a bit else it will all blow up from the mixer. Grind the masala to a smooth paste and set aside. To the same pan now add 2 tablespoons butter along with remaining 1 teaspoon oil on medium heat.Once the butter melts, add the red chili powder and the Kashmiri red chili powder and fry for few seconds. This will give the curry a nice red color. Then add the ground paste back into the pan along with the garam masala (start with 1/2 teaspoon and add the remaining 1/4 teaspoon at the end only if you feel like the curry needs that extra bit of garam masala), cardamom powder, sugar, salt and tomato paste (if using).Mix well and cook for 1-2 minutes.Then add the cream and mix. Add in the paneer and cook for 2 to 3 minutes on medium heat. Finally add crushed kasuri methi.Garnish paneer butter masala with cilantro and serve hot with naan or rice!""".replace("\n", "")
summary = "Heat oil in a pan, sauté spices, onions, garlic, ginger, then tomatoes and cashews, cook with water, blend into a paste, melt butter, add chili powders, return paste to pan with spices, sugar, salt, tomato paste, cook, add cream, paneer, kasuri methi, garnish with cilantro, serve with naan or rice."
df = pd.read_csv("./wiki_bio_gpt3_hallucination.csv")

def compute_score_per_document(scores):
    scores = ast.literal_eval(scores)
    scores = np.array(scores)
    return scores.mean()

df["average_score"] = df["sent_scores_nli"].apply(compute_score_per_document)
sorted_df = df.sort_values(by=['average_score'], ascending=False)

THRESHOLD = 0.5

examples = {}
for i in range(3):
    sample = sorted_df.iloc[[i]]
    examples[f"High hallucination sample {i+1}"] = (sample.index[0] , sample["gpt3_text"].values[0])
    sample = sorted_df.iloc[[-(i+1)]]
    examples[f"Low hallucination sample {i+1}"] = (sample.index[0] , sample["gpt3_text"].values[0])

def mirror(example):
    return examples[example][1]

def evaluate(example, treshold):
    index = examples[example][0]
    row = sorted_df.loc[index]
    
    scores = ast.literal_eval(row["sent_scores_nli"])
    sentences = ast.literal_eval(row["gpt3_sentences"])
    annotations = ast.literal_eval(row["annotation"])
    predictions = []
    labels = []

    n = len(sentences)
    average_score_predicted = 0.0
    average_score_truth = 0.0

    for score, sentence, annotation in zip(scores, sentences, annotations):
        if score > treshold:
            prediction = "hallucination"
            average_score_predicted += 1.0
        else:
            prediction = "factual"
        if annotation == "accurate":
            annotation = "factual"
        else:
            annotation = "hallucination"
            average_score_truth += 1.0
        
        predictions.append((sentence, prediction))
        labels.append((sentence, annotation))
    average_score_predicted /= n
    average_score_predicted = "{:.0%}".format(average_score_predicted)

    average_score_truth /= n
    average_score_truth = "{:.0%}".format(average_score_truth)
    return average_score_predicted, predictions, labels, average_score_truth

with gr.Blocks(theme=style) as demo:
    with gr.Tab("Maya"):
        gr.Markdown(title)
        gr.Markdown(description)
        
    with gr.Tab("Mithya"):  
        gr.Markdown(title)
        gr.Markdown(description)
        with gr.Row():
            with gr.Column():
                examples_dropdown = gr.Dropdown(choices=list(examples.keys()), value=list(examples.keys())[0],
                                        interactive=True,
                                        label="Samples",
                                        info="""You can choose among high/low hallucinations examples from Wiki Bio.
                                        More samples are available below.""")
                example_text = gr.TextArea(value=list(examples.values())[0][1])
                with gr.Accordion("Detection threshold", open=False):
                    treshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, value=THRESHOLD, label="Detection threshold", info="""The threshold used to detect hallucinations.
                                        A sentence is flagged as hallucination when inconsistency (SelfCheckGPT NLI) score is above threshold.
                                        Higher threshold increases precision (flagged hallucination actually being an hallucination) but reduces recall (percentage of hallucinations flagged).""")
                submit = gr.Button("Check hallucination", variant="primary")
            with gr.Column():
                label = gr.Label(label="Percentage of document flagged as hallucination")
                highlighted_prediction = gr.HighlightedText(
                                        label="Hallucination detection",
                                        combine_adjacent=True,
                                        color_map={"hallucination": "red", "factual": "green"},
                                        show_legend=True)
                with gr.Accordion("Ground truth", open=False):
                    gr.Markdown("Ground truth label manually annotated by humans. You can use that to compare the hallucination detection with the ground truth.")
                    label_ground_truth = gr.Label(label="Percentage of document actually hallucinations")
                    highlighted_ground_truth = gr.HighlightedText(
                                            label="Ground truth",
                                            combine_adjacent=True,
                                            color_map={"hallucination": "red", "factual": "green"},
                                            show_legend=True)
        examples_dropdown.input(mirror, inputs=examples_dropdown, outputs=example_text)
        submit.click(evaluate, inputs=[examples_dropdown, treshold], outputs=[label, highlighted_prediction, highlighted_ground_truth, label_ground_truth])
    with gr.Tab("Router-Chain-Branch"):  
        gr.Markdown(title)
        gr.Markdown(description)
        
theme=gr.themes.Base()            
demo.launch(debug=True)