feat: implementing everything for release version 1.0.0

.dockerignore

@@ -3,6 +3,10 @@ Compose.yaml
 Dockerfile-Base
 Dockerfile-Light
 entrypoint.sh
-railway.json
+.gitignore
+.github
+.git
+.pre-commit-config.yaml
+start-venv.sh
 /components/
 /components/*

README.md

@@ -17,28 +17,26 @@ app_port: 8080
 
 ## 🔗 Links:
 
-**[Github Repository](https://github.com/LennardZuendorf/thesis)**
+**[Github Repository](https://github.com/LennardZuendorf/thesis-webapp)**
+**[Huggingface Spaces Showcase](https://huggingface.co/spaces/lennardzuendorf/thesis-webapp-docker**
 
 ## 🏗️ Tech Stack:
 
-**Language and Framework:** Python, JupyterNotebook
+**Language and Framework:** Python
 
-**Noteable Packages:** 🤗 Transformers, Gradio, SHAP, BERTViz, Shapash
+**Noteable Packages:** 🤗 Transformers, FastAPI, Gradio, SHAP, BERTViz
 
 ## 👨‍💻 Author and Credits:</h2>
 
-
 **Author:** [@LennardZuendorf](https://github.com/LennardZuendorf)
 
 **Thesis Supervisor**: [Prof. Dr. Simbeck](https://www.htw-berlin.de/hochschule/personen/person/?eid=9862)
 <br> Second Corrector: [Prof. Dr. Hochstein](https://www.htw-berlin.de/hochschule/personen/person/?eid=10628)
 
+See code for in detailed credits, work is based on
 
-See code for in detail credits, work is based on
-
-- Mistral AI
+- GODEL:
 - SHAP:
 - BERTViz:
 
-
 This Project was part of my studies of Business Computing at University of Applied Science for Technology and Business Berlin (HTW Berlin).

backend/controller.py

@@ -5,17 +5,18 @@
 import gradio as gr
 
 # internal imports
-from model import mistral, godel
+from model import godel
 from explanation import interpret, visualize
 
 
 # main interference function that that calls chat functions depending on selections
+# TODO: Limit maximum tokens/model input
 def interference(
-    prompt,
-    history,
-    system_prompt,
-    model_selection,
-    xai_selection,
+    prompt: str,
+    history: list,
+    knowledge: str,
+    system_prompt: str,
+    xai_selection: str,
 ):
     # if no system prompt is given, use a default one
     if system_prompt == "":
@@ -24,20 +25,7 @@ def interference(
             Always answer as helpfully as possible, while being safe.
         """
 
-    # grabs the model instance depending on the selection
-    match model_selection.lower():
-        case "mistral":
-            model = mistral
-        case "godel":
-            model = godel
-        case _:
-            # use Gradio warning to display error message
-            gr.Warning(
-                f'There was an error in the selected model. It is "{model_selection}"'
-            )
-            raise RuntimeError("There was an error in the selected model.")
-
-    # additionally, if the XAI approach is selected, grab the XAI instance
+    # if a XAI approach is selected, grab the XAI instance
     if xai_selection in ("SHAP", "Visualizer"):
         match xai_selection.lower():
             case "shap":
@@ -46,33 +34,39 @@ def interference(
                 xai = visualize
             case _:
                 # use Gradio warning to display error message
-                gr.Warning(
-                    f"""
+                gr.Warning(f"""
                     There was an error in the selected XAI Approach.
                     It is "{xai_selection}"
-                    """
-                )
+                    """)
                 raise RuntimeError("There was an error in the selected XAI approach.")
 
         # call the explained chat function
         prompt_output, history_output, xai_graphic, xai_plot = explained_chat(
-            model=model,
+            model=godel,
             xai=xai,
             message=prompt,
             history=history,
             system_prompt=system_prompt,
+            knowledge=knowledge,
         )
     # if no (or invalid) XAI approach is selected call the vanilla chat function
     else:
         # call the vanilla chat function
         prompt_output, history_output = vanilla_chat(
-            model=model,
+            model=godel,
             message=prompt,
             history=history,
             system_prompt=system_prompt,
+            knowledge=knowledge,
         )
         # set XAI outputs to disclaimer html/none
-        xai_graphic, xai_plot = "<div><h1>No Graphic to Display</h1></div>", None
+        xai_graphic, xai_plot = (
+            """
+            <div style="text-align: center"><h4>Without Selected XAI Approach,
+            no graphic will be displayed</h4></div>
+            """,
+            None,
+        )
 
     # return the outputs
     return prompt_output, history_output, xai_graphic, xai_plot
@@ -80,27 +74,31 @@ def interference(
 
 # simple chat function that calls the model
 # formats prompts, calls for an answer and returns updated conversation history
-def vanilla_chat(model, message: str, history: list, system_prompt: str):
+def vanilla_chat(
+    model, message: str, history: list, system_prompt: str, knowledge: str = ""
+):
     # formatting the prompt using the model's format_prompt function
-    prompt = model.format_prompt(message, history, system_prompt)
+    prompt = model.format_prompt(message, history, system_prompt, knowledge)
     # generating an answer using the model's respond function
     answer = model.respond(prompt)
 
     # updating the chat history with the new answer
-    history.append((prompt, answer))
+    history.append((message, answer))
 
     # returning the updated history
     return "", history
 
 
-def explained_chat(model, xai, message: str, history: list, system_prompt: str):
+def explained_chat(
+    model, xai, message: str, history: list, system_prompt: str, knowledge: str = ""
+):
     # formatting the prompt using the model's format_prompt function
-    prompt = model.format_prompt(message, history, system_prompt)
+    prompt = model.format_prompt(message, history, system_prompt, knowledge)
 
     # generating an answer using the xai methods explain and respond function
     answer, xai_graphic, xai_plot = xai.chat_explained(model, prompt)
     # updating the chat history with the new answer
-    history.append((prompt, answer))
+    history.append((message, answer))
 
     # returning the updated history, xai graphic and xai plot elements
-    return "", [["", ""]], xai_graphic, xai_plot
+    return "", history, xai_graphic, xai_plot

explanation/interpret.py

@@ -3,40 +3,60 @@
 import seaborn as sns
 import matplotlib.pyplot as plt
 import numpy as np
-import shap
+from shap import models, maskers, plots, PartitionExplainer
+import torch
+
+# internal imports
+from utils import formatting as fmt
+
+# global variables
+TEACHER_FORCING = None
+TEXT_MASKER = None
 
 
 # main explain function that returns a chat with explanations
 def chat_explained(model, prompt):
+    model.set_config()
+
     # create the shap explainer
-    shap_explainer = shap.PartitionExplainer(model.MODEL, model.TOKENIZER)
+    shap_explainer = PartitionExplainer(model.MODEL, model.TOKENIZER)
     # get the shap values for the prompt
-    shap_values = shap_explainer(prompt)
+    shap_values = shap_explainer([prompt])
 
     # create the explanation graphic and plot
     graphic = create_graphic(shap_values)
     plot = create_plot(shap_values)
 
     # create the response text
-    response_text = format_output_text(shap_values.output_names)
+    response_text = fmt.format_output_text(shap_values.output_names)
     return response_text, graphic, plot
 
 
-# output text formatting function that turns the list into a string
-def format_output_text(output):
-    # start string with first list item
-    output_str = output[0]
-    # add all other list items with a space in between
-    for txt in output[1:]:
-        output_str += " " + txt
-    # return the output string
-    return output_str
+def wrap_shap(model):
+    global TEXT_MASKER, TEACHER_FORCING
+
+    # set the device to cuda if gpu is available
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    # updating the model settings again
+    model.set_config()
+
+    # (re)initialize the shap models and masker
+    text_generation = models.TextGeneration(model.MODEL, model.TOKENIZER)
+    TEACHER_FORCING = models.TeacherForcing(
+        text_generation,
+        model.TOKENIZER,
+        device=str(device),
+        similarity_model=model.MODEL,
+        similarity_tokenizer=model.TOKENIZER,
+    )
+    TEXT_MASKER = maskers.Text(model.TOKENIZER, " ", collapse_mask_token=True)
 
 
 # graphic plotting function that creates a html graphic (as string) for the explanation
 def create_graphic(shap_values):
     # create the html graphic using shap text plot function
-    graphic_html = shap.plots.text(shap_values, display=False)
+    graphic_html = plots.text(shap_values, display=False)
 
     # return the html graphic as string
     return str(graphic_html)
@@ -44,42 +64,59 @@ def create_graphic(shap_values):
 
 # plotting function that creates a heatmap style explanation plot
 def create_plot(shap_values):
-    # setup color palette for heatmap
-    color_palette = sns.color_palette("coolwarm", as_cmap=True)
-
-    # extract values, text from shap_values
-    values = shap_values[0]
-    input_text = shap_values.data[0]
-    output_text = shap_values.output_names
-
-    # Set the seaborn style for better aesthetics
-    sns.set(style="darkgrid")
-    plt.figure(figsize=(20, 10))
-
-    # create the heatmap with horizontal shape
-    sns.heatmap(
-        values,
-        cmap=color_palette,
-        center=0,
-        annot=False,
-        cbar_kws={"fraction": 0.02},
+    values = shap_values.values[0]
+    output_names = shap_values.output_names
+    input_names = shap_values.data[0]
+
+    # Transpose the values for horizontal input names
+    transposed_values = np.transpose(values)
+
+    # Set seaborn style to dark
+    sns.set(style="dark")
+
+    fig, ax = plt.subplots()
+
+    # Making background transparent
+    ax.set_alpha(0)
+    fig.patch.set_alpha(0)
+
+    # Setting figure size
+    fig.set_size_inches(
+        max(transposed_values.shape[1] * 2, 10),
+        max(transposed_values.shape[0] / 1.5, 5),
     )
 
-    # adjusting labels and ticks
-    plt.xticks(
-        ticks=np.arange(len(output_text)) + 0.5,
-        labels=output_text,
-        rotation=90,
+    # Plotting the heatmap with Seaborn's color palette
+    im = ax.imshow(
+        transposed_values,
+        vmax=transposed_values.max(),
+        vmin=-transposed_values.min(),
+        cmap=sns.color_palette("vlag_r", as_cmap=True),
+        aspect="auto",
     )
-    plt.yticks(
-        ticks=np.arange(len(input_text)) + 0.5,
-        labels=input_text,
-        rotation=0,
+
+    # Creating colorbar
+    cbar = ax.figure.colorbar(im, ax=ax)
+    cbar.ax.set_ylabel("Token Attribution", rotation=-90, va="bottom")
+    cbar.ax.yaxis.set_tick_params(color="white")
+    plt.setp(plt.getp(cbar.ax.axes, "yticklabels"), color="white")
+
+    # Setting ticks and labels with white color for visibility
+    ax.set_xticks(np.arange(len(input_names)), labels=input_names)
+    ax.set_yticks(np.arange(len(output_names)), labels=output_names)
+    plt.setp(ax.get_xticklabels(), color="white", rotation=45, ha="right")
+    plt.setp(ax.get_yticklabels(), color="white")
+
+    # Adjusting tick labels
+    ax.tick_params(
+        top=True, bottom=False, labeltop=False, labelbottom=True, color="white"
     )
 
-    # set axis labels
-    plt.xlabel("Output Tokens")
-    plt.ylabel("Input Tokens")
-    plt.title("Token-wise SHAP Values")
+    # Adding text annotations - not used for readability
+    # for i in range(transposed_values.shape[0]):
+    #    for j in range(transposed_values.shape[1]):
+    #        val = transposed_values[i, j]
+    #        color = "black" if 0.2 < im.norm(val) < 0.8 else "white"
+    #        ax.text(j, i, f"{val:.4f}", ha="center", va="center", color=color)
 
     return plt

explanation/visualize.py

@@ -2,20 +2,109 @@
 
 # external imports
 from bertviz import head_view
+import matplotlib.pyplot as plt
+import seaborn as sns
+import numpy as np
+
+# internal imports
+from utils import formatting as fmt
 
 
 # plotting function that plots the attention values in a heatmap
 def chat_explained(model, prompt):
-    inputs = model.TOKENIZER(prompt, return_tensors="pt")
-    out = model.MODEL(**inputs, output_attentions=True)
 
-    attention = out["attentions"]  # Retrieve attention from model outputs
-    tokens = model.TOKENIZER.convert_ids_to_tokens(
-        inputs["input_ids"][0]
-    )  # Convert input ids to token strings
+    model.set_config()
+
+    # get encoded input and output vectors
+    encoder_input_ids = model.TOKENIZER(
+        prompt, return_tensors="pt", add_special_tokens=True
+    ).input_ids
+    decoder_input_ids = model.MODEL.generate(encoder_input_ids, output_attentions=True)
+    encoder_text = fmt.format_tokens(
+        model.TOKENIZER.convert_ids_to_tokens(encoder_input_ids[0])
+    )
+    decoder_text = fmt.format_tokens(
+        model.TOKENIZER.convert_ids_to_tokens(decoder_input_ids[0])
+    )
 
-    graphic = head_view(attention, tokens)
-    response_text = out[0]
-    plot = None
+    # get attention values for the input and output vectors
+    attention_output = model.MODEL(
+        input_ids=encoder_input_ids,
+        decoder_input_ids=decoder_input_ids,
+        output_attentions=True,
+    )
 
+    # create the response text, graphic and plot
+    response_text = fmt.format_output_text(decoder_text)
+    graphic = create_graphic(attention_output, (encoder_text, decoder_text))
+    plot = create_plot(attention_output, (encoder_text, decoder_text))
     return response_text, graphic, plot
+
+
+# creating a html graphic using BERTViz
+def create_graphic(attention_output, enc_dec_texts: tuple):
+
+    # calls the head_view function of BERTViz to return html graphic
+    hview = head_view(
+        encoder_attention=attention_output.encoder_attentions,
+        decoder_attention=attention_output.decoder_attentions,
+        cross_attention=attention_output.cross_attentions,
+        encoder_tokens=enc_dec_texts[0],
+        decoder_tokens=enc_dec_texts[1],
+        html_action="return",
+    )
+
+    return str(hview.data)
+
+
+# creating an attention heatmap plot using seaborn
+def create_plot(attention_output, enc_dec_texts: tuple):
+    # get the averaged attention weights
+    attention = attention_output.cross_attentions[0][0].detach().numpy()
+    averaged_attention_weights = np.mean(attention, axis=0)
+
+    # get the encoder and decoder tokens
+    encoder_tokens = enc_dec_texts[0]
+    decoder_tokens = enc_dec_texts[1]
+
+    # set seaborn style to dark and initialize figure and axis
+    sns.set(style="dark")
+    fig, ax = plt.subplots()
+
+    # Making background transparent
+    ax.set_alpha(0)
+    fig.patch.set_alpha(0)
+
+    # Setting figure size
+    fig.set_size_inches(
+        max(averaged_attention_weights.shape[1] * 2, 10),
+        max(averaged_attention_weights.shape[0] / 1.5, 5),
+    )
+
+    # Plotting the heatmap with seaborn's color palette
+    im = ax.imshow(
+        averaged_attention_weights,
+        vmax=averaged_attention_weights.max(),
+        vmin=-averaged_attention_weights.min(),
+        cmap=sns.color_palette("rocket", as_cmap=True),
+        aspect="auto",
+    )
+
+    # Creating colorbar
+    cbar = ax.figure.colorbar(im, ax=ax)
+    cbar.ax.set_ylabel("Token Attribution", rotation=-90, va="bottom")
+    cbar.ax.yaxis.set_tick_params(color="white")
+    plt.setp(plt.getp(cbar.ax.axes, "yticklabels"), color="white")
+
+    # Setting ticks and labels with white color for visibility
+    ax.set_xticks(np.arange(len(encoder_tokens)), labels=encoder_tokens)
+    ax.set_yticks(np.arange(len(decoder_tokens)), labels=decoder_tokens)
+    plt.setp(ax.get_xticklabels(), color="white", rotation=45, ha="right")
+    plt.setp(ax.get_yticklabels(), color="white")
+
+    # Adjusting tick labels
+    ax.tick_params(
+        top=True, bottom=False, labeltop=False, labelbottom=True, color="white"
+    )
+
+    return plt

main.py

@@ -7,8 +7,9 @@ import gradio as gr
 # internal imports
 from backend.controller import interference
 
-# Global Variables
+# Global Variables and css
 app = FastAPI()
+css = "body {text-align: start !important;}"
 
 
 # different functions to provide frontend abilities
@@ -36,37 +37,33 @@ def xai_info(xai_radio):
         gr.Info("No XAI method was selected.")
 
 
-# function to display the model info
-def model_info(model_radio):
-    # display the model using the Gradio Info component
-    gr.Info(f"The model was set to:\n {model_radio}")
-
-
 # ui interface based on Gradio Blocks (see documentation:
 # https://www.gradio.app/docs/interface)
-with gr.Blocks() as ui:
+with gr.Blocks(
+    css="text-align: start !important",
+    title="Thesis Webapp Showcase",
+    head="<head>",
+) as ui:
     # header row with markdown based text
     with gr.Row():
         # markdown component to display the header
-        gr.Markdown(
-            """
-            # Thesis Demo - AI Chat Application with XAI
+        gr.Markdown("""
+            # Thesis Demo - AI Chat Application with GODEL
+            ## XAI powered by SHAP and BERTVIZ
             ### Select between tabs below for the different views.
-            """
-        )
+            """)
     # ChatBot tab used to chat with the AI chatbot
     with gr.Tab("AI ChatBot"):
         with gr.Row():
             # markdown component to display the header of the current tab
-            gr.Markdown(
-                """
+            gr.Markdown("""
                 ### ChatBot Demo
                 Chat with the AI ChatBot using the textbox below.
                 Manipulate the settings in the row above,
                 including the selection of the model,
                 the system prompt and the XAI method.
-                """
-            )
+
+                """)
         # row with columns for the different settings
         with gr.Row(equal_height=True):
             # column that takes up 3/5 of the row
@@ -80,22 +77,12 @@ with gr.Blocks() as ui:
                         " answer as helpfully as possible, while being safe."
                     ),
                 )
-            with gr.Column(scale=1):
-                # checkbox group to select the model
-                model = gr.Radio(
-                    ["Mistral", "GODEL"],
-                    label="Model Selection",
-                    info="Select Model to use for chat.",
-                    value="Mistral",
-                    interactive=True,
-                    show_label=True,
-                )
             with gr.Column(scale=1):
                 # checkbox group to select the xai method
-                xai = gr.Radio(
+                xai_selection = gr.Radio(
                     ["None", "SHAP", "Visualizer"],
                     label="XAI Settings",
-                    info="XAI Functionalities to use.",
+                    info="Select a XAI Implementation to use.",
                     value="None",
                     interactive=True,
                     show_label=True,
@@ -103,11 +90,10 @@ with gr.Blocks() as ui:
 
             # calling info functions on inputs for different settings
             system_prompt.submit(system_prompt_info, [system_prompt])
-            model.input(model_info, [model])
-            xai.input(xai_info, [xai])
+            xai_selection.input(xai_info, [xai_selection])
 
         # row with chatbot ui displaying "conversation" with the model
-        with gr.Row():
+        with gr.Row(equal_height=True):
             # out of the  box chatbot component
             # see documentation: https://www.gradio.app/docs/chatbot
             chatbot = gr.Chatbot(
@@ -115,10 +101,28 @@ with gr.Blocks() as ui:
                 show_copy_button=True,
                 avatar_images=("./public/human.jpg", "./public/bot.jpg"),
             )
-        # row with input textbox
+        # rows with input textboxes
+        with gr.Row():
+            # textbox to enter the knowledge
+            with gr.Accordion(label="Additional Knowledge", open=False):
+                knowledge_input = gr.Textbox(
+                    value="",
+                    label="Knowledge",
+                    max_lines=5,
+                    info="Add additional context knowledge.",
+                    show_label=True,
+                )
         with gr.Row():
             # textbox to enter the user prompt
-            user_prompt = gr.Textbox(label="Input Message")
+            user_prompt = gr.Textbox(
+                label="Input Message",
+                max_lines=5,
+                info="""
+                Ask the ChatBot a question.
+                Hint: More complicated question give better explanation insights!
+                """,
+                show_label=True,
+            )
         # row with columns for buttons to submit and clear content
         with gr.Row():
             with gr.Column(scale=1):
@@ -127,79 +131,84 @@ with gr.Blocks() as ui:
                 clear_btn = gr.ClearButton([user_prompt, chatbot])
             with gr.Column(scale=1):
                 submit_btn = gr.Button("Submit", variant="primary")
+        with gr.Row():
+            gr.Examples(
+                label="Example Questions",
+                examples=[
+                    [
+                        "How does a black hole form in space?",
+                        (
+                            "Black holes are created when a massive star's core"
+                            " collapses after a supernova, forming an object with"
+                            " gravity so intense that even light cannot escape."
+                        ),
+                    ],
+                    [
+                        (
+                            "Explain the importance of the Rosetta Stone in"
+                            " understanding ancient languages."
+                        ),
+                        (
+                            "The Rosetta Stone, an ancient Egyptian artifact, was key"
+                            " in decoding hieroglyphs, featuring the same text in three"
+                            " scripts: hieroglyphs, Demotic, and Greek."
+                        ),
+                    ],
+                ],
+                inputs=[user_prompt, knowledge_input],
+            )
 
     # explanations tab used to provide explanations for a specific conversation
     with gr.Tab("Explanations"):
         # row with markdown component to display the header of the current tab
         with gr.Row():
-            gr.Markdown(
-                """
+            gr.Markdown("""
                 ### Get Explanations for Conversations
                 Using your selected XAI method, you can get explanations for
                 the conversation you had with the AI ChatBot. The explanations are
                 based on the last message you sent to the AI ChatBot (see text)
-                """
-            )
-        # row that displays the settings used to create the current model output
-        ## each textbox statically displays the current values
-        with gr.Row():
-            with gr.Column():
-                gr.Textbox(
-                    value=xai,
-                    label="Used XAI Variant",
-                    show_label=True,
-                    interactive=True,
-                )
-            with gr.Column():
-                gr.Textbox(
-                    value=model, label="Used Model", show_label=True, interactive=True
-                )
-            with gr.Column():
-                gr.Textbox(
-                    value=system_prompt,
-                    label="Used System Prompt",
-                    show_label=True,
-                    interactive=True,
-                )
+                """)
         # row that displays the generated explanation of the model (if applicable)
-        with gr.Row():
-            # wraps the explanation html in an iframe to display it
+        with gr.Row(variant="panel"):
+            # wraps the explanation html in an iframe to display it interactively
             xai_interactive = gr.HTML(
                 label="Interactive Explanation",
+                value=(
+                    '<div style="text-align: center"><h4>No Graphic to Display'
+                    " (Yet)</h4></div>"
+                ),
                 show_label=True,
-                value="<div><h1>No Graphic to Display</h1></div>",
             )
         # row and accordion to display an explanation plot (if applicable)
         with gr.Row():
             with gr.Accordion("Token Explanation Plot", open=False):
+                gr.Markdown("""
+                #### Plotted Values
+                Values have been excluded for readability. See colorbar for value indication.
+                """)
                 # plot component that takes a matplotlib figure as input
-                xai_plot = gr.Plot(
-                    label="Token Level Explanation",
-                    show_label=True,
-                    every=5,
-                )
+                xai_plot = gr.Plot(label="Token Level Explanation", scale=3)
 
     # functions to trigger the controller
-    ## takes information for the chat and the model, xai selection
+    ## takes information for the chat and the xai selection
     ## returns prompt, history and xai data
     ## see backend/controller.py for more information
     submit_btn.click(
         interference,
-        [user_prompt, chatbot, system_prompt, model, xai],
+        [user_prompt, chatbot, knowledge_input, system_prompt, xai_selection],
         [user_prompt, chatbot, xai_interactive, xai_plot],
     )
     # function triggered by the enter key
     user_prompt.submit(
         interference,
-        [user_prompt, chatbot, system_prompt, model, xai],
+        [user_prompt, chatbot, knowledge_input, system_prompt, xai_selection],
         [user_prompt, chatbot, xai_interactive, xai_plot],
     )
 
     # final row to show legal information
     ## - credits, data protection and link to the License
-    with gr.Row():
-        with gr.Accordion("Credits, Data Protection and License", open=False):
-            gr.Markdown(value=load_md("public/credits_dataprotection_license.md"))
+    with gr.Tab(label="Credits, Data Protection and License"):
+        gr.Markdown(value=load_md("public/credits_dataprotection_license.md"))
 
 # mount function for fastAPI Application
 app = gr.mount_gradio_app(app, ui, path="/")

model/godel.py

@@ -1,30 +1,55 @@
 # GODEL model module for chat interaction and model instance control
+
+# external imports
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 
+# internal imports
+from utils import modelling as mdl
+
 # model and tokenizer instance
 TOKENIZER = AutoTokenizer.from_pretrained("microsoft/GODEL-v1_1-large-seq2seq")
 MODEL = AutoModelForSeq2SeqLM.from_pretrained("microsoft/GODEL-v1_1-large-seq2seq")
-GODEL_CONFIG = {"max_new_tokens": 50, "min_length": 8, "top_p": 0.9, "do_sample": True}
+CONFIG = {"max_new_tokens": 50, "min_length": 8, "top_p": 0.9, "do_sample": True}
+
+
+# TODO: Make config variable
+def set_config(config: dict = None):
+    if config is None:
+        config = {}
+
+    MODEL.config.max_new_tokens = 50
+    MODEL.config.min_length = 8
+    MODEL.config.top_p = 0.9
+    MODEL.config.do_sample = True
 
 
 # formatting class to formatting input for the model
 # CREDIT: Adapted from official interference example on Huggingface
 ## see https://huggingface.co/microsoft/GODEL-v1_1-large-seq2seq
 def format_prompt(message: str, history: list, system_prompt: str, knowledge: str = ""):
+    # user input prompt initialization
     prompt = ""
 
+    # limits the prompt elements to the maximum token count
+    message, history, system_prompt, knowledge = mdl.prompt_limiter(
+        TOKENIZER, message, history, system_prompt, knowledge
+    )
+
     # adds knowledge text if not empty
     if knowledge != "":
         knowledge = "[KNOWLEDGE] " + knowledge
 
-    history.append([message])
-    for user_prompt, bot_response in history:
-        prompt += f"EOS {user_prompt} EOS {bot_response}"
+    # adds conversation history to the prompt
+    for conversation in history:
+        prompt += f"EOS {conversation[0]} EOS {conversation[1]}"
 
-    prompt = f"{system_prompt} [CONTEXT] {prompt} {knowledge}"
+    # adds the message to the prompt
+    prompt += f" {message}"
+    # combines the entire prompt
+    full_prompt = f"{system_prompt} [CONTEXT] {prompt} {knowledge}"
 
-    # returns the full combined prompt for the model
-    return prompt
+    # returns the formatted prompt
+    return full_prompt
 
 
 # response class calling the model and returning the model output message
@@ -32,7 +57,7 @@ def format_prompt(message: str, history: list, system_prompt: str, knowledge: st
 ## see https://huggingface.co/microsoft/GODEL-v1_1-large-seq2seq
 def respond(prompt):
     input_ids = TOKENIZER(f"{prompt}", return_tensors="pt").input_ids
-    outputs = MODEL.generate(input_ids, **GODEL_CONFIG)
+    outputs = MODEL.generate(input_ids, **CONFIG)
     output = TOKENIZER.decode(outputs[0], skip_special_tokens=True)
 
     return output

model/mistral.py

@@ -1,71 +0,0 @@
-# Mistral 7B model module for chat interaction and model instance control
-
-# external imports
-from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
-import torch
-import gradio as gr
-
-# global variables for model and tokenizer, config
-MODEL = AutoModelForCausalLM.from_pretrained("mistralai/Mistral-7B-Instruct-v0.2")
-TOKENIZER = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1")
-MISTRAL_CONFIG = GenerationConfig.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1")
-
-MISTRAL_CONFIG.update(
-    **{
-        "temperature": 0.7,
-        "max_new_tokens": 50,
-        "top_p": 0.9,
-        "repetition_penalty": 1.2,
-        "do_sample": True,
-        "seed": 42,
-    }
-)
-
-
-# function to format the prompt to include chat history, message
-# CREDIT: adapted from Venkata Bhanu Teja Pallakonda in Huggingface discussions
-## see https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1/discussions/
-
-
-def format_prompt(message: str, history: list, system_prompt: str, knowledge: str = ""):
-    prompt = ""
-    if knowledge != "":
-        gr.Warning(
-            """Mistral does not support
-            additionally knowledge!"""
-        )
-
-    # if no history, use system prompt and example message
-    if len(history) == 0:
-        prompt = f"""<s>[INST] {system_prompt} [/INST] How can I help you today? </s>
-        [INST] {message} [/INST]"""
-    else:
-        # takes the very first exchange and the system prompt as base
-        for user_prompt, bot_response in history[0]:
-            prompt = (
-                f"<s>[INST] {system_prompt} {user_prompt} [/INST] {bot_response}</s>"
-            )
-
-        # takes all the following conversations and adds them as context
-        prompt += "".join(
-            f"[INST] {user_prompt} [/INST] {bot_response}</s>"
-            for user_prompt, bot_response in history[1:]
-        )
-    return prompt
-
-
-# generation class returning the model response based on the input
-# CREDIT: adapted from official Mistral Ai 7B Instruct documentation on Huggingface
-## see https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1
-def respond(prompt):
-    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-    # tokenizing inputs and configuring model
-    input_ids = TOKENIZER(f"{prompt}", return_tensors="pt")
-    model_input = input_ids.to(device)
-    MODEL.to(device)
-
-    # generating text with tokenized input, returning output
-    output_ids = MODEL.generate(model_input, generation_config=MISTRAL_CONFIG)
-    output_text = TOKENIZER.batch_decode(output_ids)
-    return output_text[0]

public/credits_dataprotection_license.md

@@ -9,19 +9,14 @@
 For full credits, please refer to the [thesis print]()
 
 ### Models
-For this project, two different models are used. Both are used through Huggingface's [transformers](https://huggingface.co/docs/transformers/index) library.
+This implementation is build on GODEL by Microsoft, Inc.
 
-##### LlaMa 2
-LlaMa 2 is an open source model by Meta Research. See [offical paper](https://arxiv.org/pdf/2307.09288.pdf) for more information.
+##### GODEL
+GODEL is an open source model by Microsoft. See [offical paper](https://arxiv.org/abs/2206.11309) for more information.
 
-- the version used in this project is LlaMa 2 7B Chat HF (HF = special version for huggingface), see [huggingface model hub](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf)
-- the model is fine-tuned for chat interactions by Meta Research
-
-##### Mistral
-Mistral is an open source model by Mistral AI. See [offical paper](https://arxiv.org/pdf/2310.06825.pdf) for more information.
-
-- the version used in this project is Mistral Instruct, see [huggingface model hub](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1)
-- the model is fine-tuned for instruction following by Mistral AI
+- the version used in this project is GODEL Large, see [huggingface model hub](https://huggingface.co/microsoft/GODEL-v1_1-large-seq2seq?text=Hey+my+name+is+Thomas%21+How+are+you%3F)
+- the model as is a generative seq2seq transformer fine tuned for goal directed dialog
+- it supports context and knowledge base inputs
 
 ### Libraries
 This project uses a number of open source libraries, only the most important ones are listed below.
@@ -29,7 +24,7 @@ This project uses a number of open source libraries, only the most important one
 ##### Shap
 This application uses a custom version of the shap library, which is available at [GitHub](https://github.com/shap/shap).
 
-- please refer to the [shap-adapter](https://github.com/LennardZuendorf/thesis-shap-adapter) repository for more information about the changes made to the library, specifically the README and CHANGES files
+- please refer to the [thesis-custom-shap](https://github.com/LennardZuendorf/thesis-custom-shap) repository for more information about the changes made to the library, specifically the README and CHANGES files
 - the shap library and the used partition SHAP explainer are based on work by Lundberg et al. (2017), see [offical paper](https://arxiv.org/pdf/1705.07874.pdf) for more information
 
 ##### BertViz
@@ -40,10 +35,11 @@ This application uses a slightly customized version of the bertviz library, whic
 
 
 # Data Protection
-This is a non-commercial project, which does not collect any personal data. The only data collected is the data you enter into the application. This data is only used to generate the explanations and is not stored anywhere.
-However, the application may be hosted with an external service (i.e. Huggingface Spaces), which may collect data. Please refer to the data protection policies of the respective service for more information.
+This is a non-commercial research project, which does not collect any personal data. The only data collected is the data you enter into the application. This data is only used to generate the explanations and is not stored anywhere.
+
+> However, the application may be hosted with an external service (i.e. Huggingface Spaces), which may collect data.
 
-If you use the "flag" feature, the data you enter will be stored in *publicly available* csv file.
+Please refer to the data protection policies of the respective service for more information. If you use the "flag" feature, the data you enter will be stored in *publicly available* csv file.
 
 
 # License

pyproject.toml

@@ -1,6 +1,8 @@
+# configuration for formatting & linting tools
 [tool.black]
 line-length = 88
 include = '\.pyi?$'
+preview = true
 exclude = '''
 /(
     \.eggs

railway.json

@@ -1,13 +0,0 @@
-{
-  "$schema": "https://railway.app/railway.schema.json",
-  "build": {
-    "builder": "DOCKERFILE",
-    "dockerfilePath": "Dockerfile"
-  },
-  "deploy": {
-    "numReplicas": 1,
-    "sleepApplication": false,
-    "restartPolicyType": "ON_FAILURE",
-    "restartPolicyMaxRetries": 10
-  }
-}

utils/formatting.py

@@ -0,0 +1,53 @@
+# formatting util module providing formatting functions for the model input and output
+
+# external imports
+import re
+
+
+# function to format the model reponse nicely
+def format_output_text(output: list):
+    # remove special tokens from list
+    formatted_output = format_tokens(output)
+
+    # start string with first list item if it is not empty
+    if formatted_output[0] != "":
+        output_str = formatted_output[0]
+    else:
+        # alternatively start with second list item
+        output_str = formatted_output[1]
+
+    # add all other list items with a space in between
+    for txt in formatted_output[1:]:
+        # check if the token is a punctuation mark
+        if txt in [".", ",", "!", "?"]:
+            # add punctuation mark without space
+            output_str += txt
+        # add token with space if not empty
+        elif txt != "":
+            output_str += " " + txt
+
+    # return the combined string with multiple spaces removed
+    return re.sub(" +", " ", output_str)
+
+
+# format the tokens by removing special tokens and special characters
+def format_tokens(tokens: list):
+    # define special tokens to remove and initialize empty list
+    special_tokens = ["[CLS]", "[SEP]", "[PAD]", "[UNK]", "[MASK]", "▁", "Ġ", "</w>"]
+    updated_tokens = []
+
+    # loop through tokens
+    for t in tokens:
+        # remove special token from start of token if found
+        if t.startswith("▁"):
+            t = t.lstrip("▁")
+
+        # loop through special tokens and remove them if found
+        for s in special_tokens:
+            t = t.replace(s, "")
+
+        # add token to list
+        updated_tokens.append(t)
+
+    # return the list of tokens
+    return updated_tokens

utils/modelling.py

@@ -0,0 +1,69 @@
+# module for modelling utilities
+
+# external imports
+import gradio as gr
+
+
+def prompt_limiter(
+    tokenizer, message: str, history: list, system_prompt: str, knowledge: str = ""
+):
+    # initializing the prompt history empty
+    prompt_history = []
+    # getting the token count for the message, system prompt, and knowledge
+    pre_count = (
+        token_counter(tokenizer, message)
+        + token_counter(tokenizer, system_prompt)
+        + token_counter(tokenizer, knowledge)
+    )
+
+    # validating the token count
+    # check if token count already too high
+    if pre_count > 1024:
+
+        # check if token count too high even without knowledge
+        if (
+            token_counter(tokenizer, message) + token_counter(tokenizer, system_prompt)
+            > 1024
+        ):
+
+            # show warning and raise error
+            gr.Warning("Message and system prompt are too long. Please shorten them.")
+            raise RuntimeError(
+                "Message and system prompt are too long. Please shorten them."
+            )
+
+        # show warning and remove knowledge
+        gr.Warning("Knowledge is too long. It has been removed to keep model running.")
+        return message, prompt_history, system_prompt, ""
+
+    # if token count small enough, add history
+    if pre_count < 800:
+        # setting the count to the precount
+        count = pre_count
+        # reversing the history to prioritize recent conversations
+        history.reverse()
+
+        # iterating through the history
+        for conversation in history:
+
+            # checking the token count with the current conversation
+            count += token_counter(tokenizer, conversation[0]) + token_counter(
+                tokenizer, conversation[1]
+            )
+
+            # add conversation or break loop depending on token count
+            if count < 1024:
+                prompt_history.append(conversation)
+            else:
+                break
+
+    # return the message, prompt history, system prompt, and knowledge
+    return message, prompt_history, system_prompt, knowledge
+
+
+# token counter function using the model tokenizer
+def token_counter(tokenizer, text: str):
+    # tokenize the text
+    tokens = tokenizer(text, return_tensors="pt").input_ids
+    # return the token count
+    return len(tokens[0])