Update faiss DB with better chunking, added tabs and visualization

app.py

@@ -3,6 +3,8 @@ from huggingface_hub import login, InferenceClient
 import os
 from langchain_community.vectorstores import FAISS
 from langchain_community.embeddings import HuggingFaceEmbeddings
+import umap
+import pandas as pd
 
 HF_TOKEN = os.getenv("HUGGINGFACE_TOKEN")
 
@@ -16,6 +18,18 @@ db_code = FAISS.load_local("faiss_code_education",
         embeddings,
         allow_dangerous_deserialization=True)
 
+reducer = umap.UMAP()
+index = db_code.index
+ntotal = min(index.ntotal, 4998)
+embeds = index.reconstruct_n(0, ntotal)
+umap_embeds = reducer.fit_transform(embeds)
+
+articles_df = pd.DataFrame({
+    "x" : umap_embeds[:,0],
+    "y" : umap_embeds[:,1],
+    "type" : [ "Source" ] * len(umap_embeds),
+})
+
 system_prompt = """Tu es un assistant juridique spécialisé dans le Code de l'éducation français. 
 Ta mission est d'aider les utilisateurs à comprendre la législation en répondant à leurs questions.
 
@@ -45,6 +59,9 @@ def query_rag(query, model, system_prompt):
     messages = [ { "role" : "system", "content" : system_prompt } ]
     messages.append( { "role" : "user", "content" : user } )
 
+    if "factice" in model:
+        return user, article_dict
+
     chat_completion = client.chat_completion(
         messages=messages,
         model=model,
@@ -53,17 +70,27 @@ def query_rag(query, model, system_prompt):
     return chat_completion.choices[0].message.content, article_dict
 
 def create_context_response(response, article_dict):
-    response += '\n\n**Références**\n\n'
+    context = '\n'
     for i, article in enumerate(article_dict):
         art = article_dict[article]
-        response += '* **' + art['chemin'] + '** : '+ art['texte'].replace('\n', '\n    ')+'\n'
-
-    return response
+        context += '* **' + art['chemin'] + '** : '+ art['texte'].replace('\n', '\n    ')+'\n'
+    return context
 
 def chat_interface(query, model, system_prompt):
     response, article_dict = query_rag(query, model, system_prompt)
-    response_with_context = create_context_response(response, article_dict)
-    return response_with_context
+    context = create_context_response(response, article_dict)
+    return response, context
+
+def update_plot(query):
+    query_embed = embeddings.embed_documents([query])[0]
+    query_umap_embed = reducer.transform([query_embed])
+    
+    data = {
+        "x": umap_embeds[:, 0].tolist() + [query_umap_embed[0, 0]],
+        "y": umap_embeds[:, 1].tolist() + [query_umap_embed[0, 1]],
+        "type": ["Source"] * len(umap_embeds) + ["Requête"]
+    }
+    return pd.DataFrame(data)
 
 with gr.Blocks(title="Assistant Juridique pour le Code de l'éducation (Beta)") as demo:
     gr.Markdown(
@@ -87,18 +114,30 @@ with gr.Blocks(title="Assistant Juridique pour le Code de l'éducation (Beta)")
             "meta-llama/Meta-Llama-3-8B-Instruct",
             "HuggingFaceH4/zephyr-7b-beta",
             "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
-            "mistralai/Mixtral-8x22B-v0.1"
+            "mistralai/Mixtral-8x22B-v0.1",
+            "factice: question+contexte"
             ],
         value="meta-llama/Meta-Llama-3-70B-Instruct")
 
     submit_button = gr.Button("Envoyer")
 
-    response_box = gr.Markdown()
-
-    system_box = gr.Textbox(label="Invite systeme", value=system_prompt)
+    with gr.Tab(label="Réponse"):
+        response_box = gr.Markdown()
+    with gr.Tab(label="Sources"):
+        sources_box = gr.Markdown()
+    with gr.Tab(label="Visualisation"):
+        scatter_plot = gr.ScatterPlot(articles_df,
+                x = "x", y = "y",
+                color="type",
+                label="Visualisation des embeddings",
+                height=500)
+    with gr.Tab(label="Paramètres"):
+        system_box = gr.Textbox(label="Invite systeme", value=system_prompt,
+                                lines=20)
 
     submit_button.click(chat_interface, 
                 inputs=[query_box, model, system_box], 
-                outputs=[response_box])
+                outputs=[response_box, sources_box])
+    submit_button.click(update_plot, inputs=[query_box], outputs=[scatter_plot])
 
 demo.launch()

faiss_code_education/index.faiss

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c2df7172919daa30d4dcb6b540cc96d5f5737da11588a0a53f09feb7391d6a2
-size 27717677
+oid sha256:9b7eeb956ab6ac2e4a131002847ea78318d7af3574dc73ac8cccc76f12424d13
+size 21831725

faiss_code_education/index.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8101642c4967dbc8bc0977b2a3b53cde856a00cd4381f490d6232954fba077d
-size 13271108
+oid sha256:e796a35535640aa94b8521f629c08c40b9b04892c8dcd40a15459abf0833fe8e
+size 6466363