geronimo-pericoli
commited on
Commit
•
b599c44
1
Parent(s):
47dbd9a
Create app.py
Browse files
app.py
ADDED
@@ -0,0 +1,674 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import nest_asyncio
|
2 |
+
nest_asyncio.apply()
|
3 |
+
|
4 |
+
from llama_index.core import (
|
5 |
+
VectorStoreIndex,
|
6 |
+
ServiceContext,
|
7 |
+
SimpleDirectoryReader,
|
8 |
+
load_index_from_storage,
|
9 |
+
)
|
10 |
+
|
11 |
+
from llama_index.core.storage import StorageContext
|
12 |
+
from llama_index.core.node_parser import SentenceSplitter
|
13 |
+
from llama_index.core.prompts import PromptTemplate
|
14 |
+
from llama_index.core.response_synthesizers import TreeSummarize
|
15 |
+
from llama_index.core.query_pipeline import InputComponent
|
16 |
+
from llama_index.core.indices.knowledge_graph import KGTableRetriever
|
17 |
+
from llama_index.legacy.vector_stores.faiss import FaissVectorStore
|
18 |
+
from llama_index.llms.openai import OpenAI
|
19 |
+
from llama_index.embeddings.openai import OpenAIEmbedding
|
20 |
+
from llama_index.core import Settings
|
21 |
+
|
22 |
+
import openai
|
23 |
+
import os
|
24 |
+
from github import Github
|
25 |
+
from datetime import datetime
|
26 |
+
import gradio as gr
|
27 |
+
import pandas as pd
|
28 |
+
|
29 |
+
|
30 |
+
|
31 |
+
# Context:
|
32 |
+
exec(os.environ.get('context'))
|
33 |
+
|
34 |
+
|
35 |
+
|
36 |
+
##### Graph start ##########
|
37 |
+
import networkx as nx
|
38 |
+
import matplotlib.pyplot as plt
|
39 |
+
from PIL import Image
|
40 |
+
from io import BytesIO
|
41 |
+
|
42 |
+
def draw_graph():
|
43 |
+
global kg_data
|
44 |
+
G = nx.DiGraph()
|
45 |
+
for source, relation, target in kg_data:
|
46 |
+
G.add_edge(source, target, label=relation)
|
47 |
+
|
48 |
+
# Utilizar spring_layout para mejorar la disposición de los nodos
|
49 |
+
pos = nx.spring_layout(G)
|
50 |
+
|
51 |
+
plt.figure(figsize=(12, 8))
|
52 |
+
# Ajustar el tamaño de los nodos
|
53 |
+
nx.draw(G, pos, with_labels=True, node_color='skyblue', node_size=400, edge_color='k', linewidths=1, font_size=8, font_weight='bold')
|
54 |
+
# Ajustar el tamaño de las flechas y el espaciado entre ellas
|
55 |
+
edge_labels = {}
|
56 |
+
for source, target, data in G.edges(data=True):
|
57 |
+
if 'label' in data:
|
58 |
+
edge_labels[(source, target)] = data['label']
|
59 |
+
nx.draw_networkx_edge_labels(G, pos, edge_labels=edge_labels, font_size=7, font_weight='normal')
|
60 |
+
plt.title("Graph")
|
61 |
+
plt.axis('off')
|
62 |
+
|
63 |
+
buf = BytesIO()
|
64 |
+
plt.savefig(buf, format='png')
|
65 |
+
buf.seek(0)
|
66 |
+
plt.close()
|
67 |
+
|
68 |
+
return Image.open(buf)
|
69 |
+
##### Graph end ##########
|
70 |
+
|
71 |
+
|
72 |
+
|
73 |
+
##### Refs start ##########
|
74 |
+
import re
|
75 |
+
def extraer_informacion_metadata(respuesta, max_results=10):
|
76 |
+
# Obtener source_nodes de la respuesta
|
77 |
+
source_nodes = respuesta.source_nodes
|
78 |
+
|
79 |
+
# Obtener page_labels, file_names y scores de source_nodes
|
80 |
+
page_file_info = [
|
81 |
+
f"<strong>Página {node.node.metadata.get('page_label', '')} del archivo {node.node.metadata.get('file_name', '')}</strong> (Relevance: {node.score:.6f} - Id: {node.node.id_})\n\n"
|
82 |
+
for node in source_nodes if node.score <= 1 # Excluir nodos con score > 1
|
83 |
+
]
|
84 |
+
|
85 |
+
# Limitar la cantidad de resultados
|
86 |
+
page_file_info = page_file_info[:max_results]
|
87 |
+
|
88 |
+
return page_file_info
|
89 |
+
|
90 |
+
|
91 |
+
import html
|
92 |
+
def extraer_textos_metadata(respuesta, max_results=10):
|
93 |
+
# Obtener source_nodes de la respuesta
|
94 |
+
source_nodes = respuesta.source_nodes
|
95 |
+
|
96 |
+
# Obtener información de página, archivo y texto de cada nodo
|
97 |
+
page_file_text_info = []
|
98 |
+
for node in source_nodes:
|
99 |
+
if node.score <= 1: # Excluir nodos con score > 1
|
100 |
+
page_label = node.node.metadata.get('page_label', '')
|
101 |
+
file_name = node.node.metadata.get('file_name', '')
|
102 |
+
text = node.node.text.strip()
|
103 |
+
|
104 |
+
# Escapar caracteres especiales en el texto
|
105 |
+
escaped_text = html.escape(text)
|
106 |
+
|
107 |
+
# Formatear con HTML
|
108 |
+
formatted_text = f"""
|
109 |
+
<div style="margin-bottom: 10px;">
|
110 |
+
<strong style="font-size: 11px;">Página {page_label} del archivo {file_name}</strong>
|
111 |
+
<p style="font-size: 9px; margin-top: 5px;">{escaped_text}</p>
|
112 |
+
<hr style="border: 1px solid #ccc; margin:10px 0;">
|
113 |
+
</div>
|
114 |
+
"""
|
115 |
+
page_file_text_info.append(formatted_text.strip()) # Quitar espacios adicionales
|
116 |
+
|
117 |
+
# Limitar la cantidad de resultados
|
118 |
+
page_file_text_info = page_file_text_info[:max_results]
|
119 |
+
|
120 |
+
return ''.join(page_file_text_info) # Devolver como un string limpio
|
121 |
+
##### Refs end ##########
|
122 |
+
|
123 |
+
|
124 |
+
|
125 |
+
##### Logs start ##########
|
126 |
+
import pandas as pd
|
127 |
+
from datasets import load_dataset, Dataset, DatasetDict
|
128 |
+
from huggingface_hub import login, HfApi, file_exists, hf_hub_download, list_repo_files
|
129 |
+
|
130 |
+
# HuggingFace Token:
|
131 |
+
HF_TOKEN = os.environ.get('hf')
|
132 |
+
|
133 |
+
# Definiciones
|
134 |
+
repo_name = "pharma-IA"
|
135 |
+
project_id = "gmpcolombia"
|
136 |
+
|
137 |
+
def save_to_dataset(user_message, response_text, user):
|
138 |
+
current_month = datetime.now().strftime('%Y-%m')
|
139 |
+
filename = f"logs_{current_month}.csv"
|
140 |
+
repo_id = f"{repo_name}/logs-{project_id}"
|
141 |
+
|
142 |
+
if file_exists(repo_id=repo_id, filename=f"{filename}", repo_type="dataset", token=HF_TOKEN):
|
143 |
+
local_filepath = hf_hub_download(
|
144 |
+
repo_id=repo_id,
|
145 |
+
filename=f"{filename}",
|
146 |
+
repo_type="dataset",
|
147 |
+
token=HF_TOKEN
|
148 |
+
)
|
149 |
+
df = pd.read_csv(local_filepath)
|
150 |
+
else:
|
151 |
+
df = pd.DataFrame(columns=["timestamp", "user_message", "response_text", "flag", "user"])
|
152 |
+
|
153 |
+
timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
154 |
+
new_data = pd.DataFrame([{
|
155 |
+
"timestamp": timestamp,
|
156 |
+
"user_message": user_message,
|
157 |
+
"response_text": response_text,
|
158 |
+
"flag": "",
|
159 |
+
"user": user
|
160 |
+
}])
|
161 |
+
|
162 |
+
df = pd.concat([df, new_data], ignore_index=True)
|
163 |
+
df.to_csv(filename, index=False)
|
164 |
+
|
165 |
+
api = HfApi()
|
166 |
+
api.upload_file(
|
167 |
+
path_or_fileobj=filename,
|
168 |
+
path_in_repo=f"{filename}",
|
169 |
+
repo_id=repo_id,
|
170 |
+
token=HF_TOKEN,
|
171 |
+
repo_type="dataset"
|
172 |
+
)
|
173 |
+
|
174 |
+
def normalize_text(text):
|
175 |
+
return text.strip().lower()
|
176 |
+
|
177 |
+
def print_like_dislike(x: gr.LikeData):
|
178 |
+
#print(f"Value: {x.value}")
|
179 |
+
#print(f"Liked: {x.liked}")
|
180 |
+
|
181 |
+
if x is not None:
|
182 |
+
text_value = x.value if isinstance(x.value, str) else x.value.get('value', '')
|
183 |
+
current_month = datetime.now().strftime('%Y-%m')
|
184 |
+
filename = f"logs_{current_month}.csv"
|
185 |
+
repo_id = f"{repo_name}/logs-{project_id}"
|
186 |
+
|
187 |
+
if file_exists(repo_id=repo_id, filename=f"{filename}", repo_type="dataset", token=HF_TOKEN):
|
188 |
+
local_filepath = hf_hub_download(
|
189 |
+
repo_id=repo_id,
|
190 |
+
filename=f"{filename}",
|
191 |
+
repo_type="dataset",
|
192 |
+
token=HF_TOKEN
|
193 |
+
)
|
194 |
+
df = pd.read_csv(local_filepath)
|
195 |
+
#print(df.head()) # Verifica el contenido del archivo CSV
|
196 |
+
|
197 |
+
normalized_value = normalize_text(text_value)
|
198 |
+
df['normalized_response_text'] = df['response_text'].apply(normalize_text)
|
199 |
+
|
200 |
+
response_indices = df.index[df['normalized_response_text'].str.contains(normalized_value, na=False, regex=False)].tolist()
|
201 |
+
print(f"Response Indices: {response_indices}")
|
202 |
+
|
203 |
+
if response_indices:
|
204 |
+
response_index = response_indices[-1]
|
205 |
+
print(f"Updating index: {response_index} with value: {x.liked}")
|
206 |
+
|
207 |
+
# Solo actualiza el valor de 'flag'
|
208 |
+
df['flag'] = df['flag'].astype(object)
|
209 |
+
df.at[response_index, 'flag'] = str(x.liked)
|
210 |
+
|
211 |
+
df = df.drop(columns=['normalized_response_text'])
|
212 |
+
df.to_csv(filename, index=False)
|
213 |
+
|
214 |
+
api = HfApi()
|
215 |
+
api.upload_file(
|
216 |
+
path_or_fileobj=filename,
|
217 |
+
path_in_repo=f"{filename}",
|
218 |
+
repo_id=repo_id,
|
219 |
+
token=HF_TOKEN,
|
220 |
+
repo_type="dataset"
|
221 |
+
)
|
222 |
+
else:
|
223 |
+
print("No matching response found to update.")
|
224 |
+
else:
|
225 |
+
print(f"File {filename} does not exist in the repository.")
|
226 |
+
else:
|
227 |
+
print("x is None.")
|
228 |
+
|
229 |
+
def save_evals_to_dataset(query, faithfulness_score, ans_relevancy_score, ctx_relevancy_score):
|
230 |
+
current_month = datetime.now().strftime('%Y-%m')
|
231 |
+
filename = f"logs_{current_month}.csv"
|
232 |
+
repo_id = f"{repo_name}/logs-{project_id}"
|
233 |
+
|
234 |
+
if file_exists(repo_id=repo_id, filename=f"{filename}", repo_type="dataset", token=HF_TOKEN):
|
235 |
+
local_filepath = hf_hub_download(
|
236 |
+
repo_id=repo_id,
|
237 |
+
filename=f"{filename}",
|
238 |
+
repo_type="dataset",
|
239 |
+
token=HF_TOKEN
|
240 |
+
)
|
241 |
+
df = pd.read_csv(local_filepath)
|
242 |
+
else:
|
243 |
+
print(f"File {filename} does not exist in the repository.")
|
244 |
+
return
|
245 |
+
|
246 |
+
# Normalizamos el query para la comparación
|
247 |
+
normalized_query = normalize_text(query).lower() # Convertimos a minúsculas
|
248 |
+
|
249 |
+
# Buscamos la última entrada que coincida con el query, convirtiendo a minúsculas en la comparación
|
250 |
+
matching_indices = df.index[df['user_message'].str.lower().str.contains(normalized_query, na=False, regex=False)].tolist()
|
251 |
+
|
252 |
+
if matching_indices:
|
253 |
+
last_index = matching_indices[-1] # Tomamos la última coincidencia
|
254 |
+
# Agregamos los puntajes a las columnas correspondientes
|
255 |
+
df.at[last_index, 'groundedness'] = faithfulness_score
|
256 |
+
df.at[last_index, 'answer_rel'] = ans_relevancy_score
|
257 |
+
df.at[last_index, 'context_rel'] = ctx_relevancy_score
|
258 |
+
|
259 |
+
df.to_csv(filename, index=False)
|
260 |
+
|
261 |
+
api = HfApi()
|
262 |
+
api.upload_file(
|
263 |
+
path_or_fileobj=filename,
|
264 |
+
path_in_repo=f"{filename}",
|
265 |
+
repo_id=repo_id,
|
266 |
+
token=HF_TOKEN,
|
267 |
+
repo_type="dataset"
|
268 |
+
)
|
269 |
+
else:
|
270 |
+
print("No matching query found in the dataset.")
|
271 |
+
|
272 |
+
# Función para verificar si un archivo existe en el repositorio
|
273 |
+
def file_exists(repo_id, filename, repo_type="dataset", token=None):
|
274 |
+
files = list_repo_files(repo_id=repo_id, repo_type=repo_type, token=token)
|
275 |
+
return filename in files
|
276 |
+
|
277 |
+
# Función para cargar las hojas de auditoría disponibles
|
278 |
+
def load_available_logs():
|
279 |
+
repo_id = f"{repo_name}/logs-{project_id}"
|
280 |
+
files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=HF_TOKEN)
|
281 |
+
|
282 |
+
# Filtramos los archivos CSV con formato 'logs_YYYY-MM.csv'
|
283 |
+
available_months = [f.split('_')[1].replace('.csv', '') for f in files if f.startswith('logs_')]
|
284 |
+
return available_months
|
285 |
+
|
286 |
+
# Cargar los logs del mes seleccionado
|
287 |
+
def load_audit_trail(selected_month):
|
288 |
+
filename = f"logs_{selected_month}.csv"
|
289 |
+
repo_id = f"{repo_name}/logs-{project_id}"
|
290 |
+
|
291 |
+
if file_exists(repo_id=repo_id, filename=filename, repo_type="dataset", token=HF_TOKEN):
|
292 |
+
local_filepath = hf_hub_download(
|
293 |
+
repo_id=repo_id,
|
294 |
+
filename=filename,
|
295 |
+
repo_type="dataset",
|
296 |
+
token=HF_TOKEN
|
297 |
+
)
|
298 |
+
df = pd.read_csv(local_filepath)
|
299 |
+
|
300 |
+
# Convertir el campo 'timestamp' a una cadena con el formato UTC-0
|
301 |
+
df["timestamp"] = pd.to_datetime(df["timestamp"]).dt.strftime('%Y-%m-%d %H:%M:%S UTC-0')
|
302 |
+
|
303 |
+
# Ordenar por la columna timestamp de forma descendente
|
304 |
+
df = df.sort_values(by="timestamp", ascending=False)
|
305 |
+
|
306 |
+
# Renombrar las columnas para visualización
|
307 |
+
df = df.rename(columns={
|
308 |
+
"timestamp": "Marca de Tiempo",
|
309 |
+
"user_message": "Mensaje Usuario",
|
310 |
+
"response_text": "Respuesta",
|
311 |
+
"flag": "Etiqueta",
|
312 |
+
"user": "Usuario",
|
313 |
+
"groundedness": "Groundedness",
|
314 |
+
"answer_rel": "Answer Relev.",
|
315 |
+
"context_rel": "Context Relev."
|
316 |
+
})
|
317 |
+
|
318 |
+
return df
|
319 |
+
else:
|
320 |
+
return pd.DataFrame(columns=["Marca de Tiempo", "Mensaje Usuario", "Respuesta", "Etiqueta", "Usuario"])
|
321 |
+
##### Logs end ##########
|
322 |
+
|
323 |
+
|
324 |
+
|
325 |
+
##### Evaluate start ##########
|
326 |
+
from llama_index.core.evaluation import FaithfulnessEvaluator
|
327 |
+
from llama_index.core.evaluation import RelevancyEvaluator
|
328 |
+
from llama_index.core.evaluation import AnswerRelevancyEvaluator
|
329 |
+
from llama_index.core.evaluation import ContextRelevancyEvaluator
|
330 |
+
|
331 |
+
final_response = ""
|
332 |
+
query = ""
|
333 |
+
|
334 |
+
def ctx_relevancy_eval():
|
335 |
+
global final_response
|
336 |
+
global query
|
337 |
+
|
338 |
+
# Verificamos si 'final_response' tiene el atributo 'source_nodes'
|
339 |
+
if not hasattr(final_response, 'source_nodes'):
|
340 |
+
raise AttributeError("El objeto 'final_response' no tiene un atributo 'source_nodes'.")
|
341 |
+
|
342 |
+
# Obtener los source_nodes de la respuesta
|
343 |
+
source_nodes = final_response.source_nodes
|
344 |
+
|
345 |
+
# Extraer los textos de los source nodes
|
346 |
+
contexts = []
|
347 |
+
for node in source_nodes:
|
348 |
+
if node.score <= 1: # Excluir nodos con score > 1
|
349 |
+
text = node.node.text.strip()
|
350 |
+
contexts.append(text)
|
351 |
+
|
352 |
+
# Si no se encuentran textos en los source nodes, puedes manejarlo de alguna forma
|
353 |
+
if not contexts:
|
354 |
+
raise ValueError("No se encontraron textos en los source nodes.")
|
355 |
+
|
356 |
+
evaluator = ContextRelevancyEvaluator(llm=gpt4omini)
|
357 |
+
evaluation_result = evaluator.evaluate(query=query, contexts=contexts)
|
358 |
+
|
359 |
+
# Extraer el puntaje de la evaluación
|
360 |
+
relevancy_score = evaluation_result.score
|
361 |
+
|
362 |
+
print("Context Relevancy: " + str(relevancy_score) + "\n")
|
363 |
+
print("Nodes: " + str(contexts) + "\n")
|
364 |
+
print("Query: " + str(query) + "\n\n----------")
|
365 |
+
return relevancy_score
|
366 |
+
|
367 |
+
def faithfulness_eval():
|
368 |
+
global final_response
|
369 |
+
evaluator = FaithfulnessEvaluator(llm=gpt4omini)
|
370 |
+
eval_result = evaluator.evaluate_response(response=final_response)
|
371 |
+
|
372 |
+
print("Groundedness: " + str(eval_result.score) + " - " + str(eval_result.passing) + "\n")
|
373 |
+
print("Respuesta: " + str(final_response) + "\n\n----------")
|
374 |
+
return float(eval_result.score)
|
375 |
+
|
376 |
+
def ans_relevancy_eval():
|
377 |
+
global final_response
|
378 |
+
global query
|
379 |
+
evaluator = AnswerRelevancyEvaluator(llm=gpt4omini)
|
380 |
+
eval_result = evaluator.evaluate(query=query, response=final_response.response)
|
381 |
+
|
382 |
+
print("Answer Relevancy: " + str(eval_result.score) + " - " + str(eval_result.passing) + "\n")
|
383 |
+
print("Respuesta: " + str(final_response.response) + "\n")
|
384 |
+
print("Query: " + str(query) + "\n\n----------")
|
385 |
+
return float(eval_result.score)
|
386 |
+
|
387 |
+
def evaluate():
|
388 |
+
global query
|
389 |
+
|
390 |
+
# Evaluaciones y escalado
|
391 |
+
faithfulness_score = round(faithfulness_eval() * 5, 1) # Redondear a un decimal
|
392 |
+
ans_relevancy_score = round(ans_relevancy_eval() * 5, 1) # Redondear a un decimal
|
393 |
+
ctx_relevancy_score = round(ctx_relevancy_eval() * 5, 1) # Redondear a un decimal
|
394 |
+
|
395 |
+
# Llamamos a save_evals_to_dataset
|
396 |
+
save_evals_to_dataset(query, faithfulness_score, ans_relevancy_score, ctx_relevancy_score)
|
397 |
+
|
398 |
+
def get_color(value):
|
399 |
+
if value <= 1.6667:
|
400 |
+
return '#f07b61' # Rojo
|
401 |
+
elif value <= 3.3334:
|
402 |
+
return '#f3e076' # Amarillo
|
403 |
+
else:
|
404 |
+
return '#84fa57' # Verde
|
405 |
+
|
406 |
+
color1 = get_color(faithfulness_score)
|
407 |
+
color2 = get_color(ans_relevancy_score)
|
408 |
+
color3 = get_color(ctx_relevancy_score)
|
409 |
+
|
410 |
+
html_output = f"""
|
411 |
+
<div style="display: flex; justify-content: space-around; align-items: center; width: 100%; flex-direction: column; text-align: center; margin: 10px 0;">
|
412 |
+
<div style="display: flex; justify-content: space-around; align-items: center; width: 100%;">
|
413 |
+
<div style="margin: 10px; display: flex; flex-direction: column; align-items: center;">
|
414 |
+
<div style="width: 50px; height: 50px; background-color: {color1}; border-radius: 50%; display: flex; justify-content: center; align-items: center; color: white; font-weight: bold;">
|
415 |
+
{faithfulness_score}
|
416 |
+
</div>
|
417 |
+
<div style="margin-top: 6px;">Groundedness</div>
|
418 |
+
</div>
|
419 |
+
<div style="margin: 10px; display: flex; flex-direction: column; align-items: center;">
|
420 |
+
<div style="width: 50px; height: 50px; background-color: {color2}; border-radius: 50%; display: flex; justify-content: center; align-items: center; color: white; font-weight: bold;">
|
421 |
+
{ans_relevancy_score}
|
422 |
+
</div>
|
423 |
+
<div style="margin-top: 6px;">Answer Relevance</div>
|
424 |
+
</div>
|
425 |
+
<div style="margin: 10px; display: flex; flex-direction: column; align-items: center;">
|
426 |
+
<div style="width: 50px; height: 50px; background-color: {color3}; border-radius: 50%; display: flex; justify-content: center; align-items: center; color: white; font-weight: bold;">
|
427 |
+
{ctx_relevancy_score}
|
428 |
+
</div>
|
429 |
+
<div style="margin-top: 6px;">Context Relevance</div>
|
430 |
+
</div>
|
431 |
+
</div>
|
432 |
+
</div>
|
433 |
+
"""
|
434 |
+
return html_output
|
435 |
+
##### Evaluate end ##########
|
436 |
+
|
437 |
+
|
438 |
+
|
439 |
+
|
440 |
+
|
441 |
+
|
442 |
+
|
443 |
+
|
444 |
+
chat_history_engine = []
|
445 |
+
result_metadata = ""
|
446 |
+
result_texts = ""
|
447 |
+
result_evals = ""
|
448 |
+
|
449 |
+
css = """
|
450 |
+
.block {
|
451 |
+
background: rgba(245, 247, 249, 0.7) !important;
|
452 |
+
}
|
453 |
+
#component-2 {
|
454 |
+
background: transparent !important;
|
455 |
+
}
|
456 |
+
|
457 |
+
.block.accordion > button:first-of-type span {
|
458 |
+
font-size: medium !important;
|
459 |
+
font-weight: bold !important;
|
460 |
+
}
|
461 |
+
|
462 |
+
.examples .block {
|
463 |
+
background: transparent !important;
|
464 |
+
}
|
465 |
+
|
466 |
+
table {
|
467 |
+
font-size: x-small !important;
|
468 |
+
}
|
469 |
+
"""
|
470 |
+
with gr.Blocks(theme='sudeepshouche/minimalist', css=css) as demo:
|
471 |
+
|
472 |
+
def get_ref():
|
473 |
+
return {mkdn: gr.Markdown(result_metadata), texts: gr.HTML(str(result_texts))}
|
474 |
+
|
475 |
+
def get_logs(selected_month):
|
476 |
+
df = load_audit_trail(selected_month)
|
477 |
+
return df
|
478 |
+
|
479 |
+
def get_evals():
|
480 |
+
global result_evals
|
481 |
+
global final_response
|
482 |
+
global query
|
483 |
+
|
484 |
+
# Verificar si 'final_response' está vacío
|
485 |
+
if not final_response:
|
486 |
+
# Si no hay final_response pero hay un resultado previo en result_evals, devolverlo
|
487 |
+
if result_evals:
|
488 |
+
return {evals: gr.HTML(f"""
|
489 |
+
<div style="display: flex; justify-content: space-around; align-items: center; width: 100%; flex-direction: column; text-align: center; margin: 10px 0;">
|
490 |
+
{result_evals}
|
491 |
+
<p style="font-size: 10px;">Esta evaluación corresponde a la consulta: <strong>{query}</strong></p>
|
492 |
+
</div>
|
493 |
+
""")}
|
494 |
+
|
495 |
+
# Si no hay final_response ni resultados previos, mostrar advertencia
|
496 |
+
gr.Info("Se necesita una respuesta completa para iniciar la evaluación.")
|
497 |
+
return {evals: gr.HTML(f"""
|
498 |
+
<div style="display: flex; justify-content: space-around; align-items: center; width: 100%; flex-direction: column; text-align: center; margin: 10px 0;">
|
499 |
+
Se necesita una respuesta completa para iniciar la evaluación.</div>
|
500 |
+
""")}
|
501 |
+
|
502 |
+
# Ejecuta la evaluación si final_response está disponible
|
503 |
+
result_evals = evaluate()
|
504 |
+
|
505 |
+
# Reiniciar 'final_response' después de la evaluación
|
506 |
+
final_response = ""
|
507 |
+
|
508 |
+
# Devolver el resultado de la evaluación
|
509 |
+
return {evals: gr.HTML(f"""{result_evals}"""), eval_accord: gr.Accordion(elem_classes="accordion", label="Evaluaciones", open=True)}
|
510 |
+
|
511 |
+
def refresh(chat_history):
|
512 |
+
global kg_data
|
513 |
+
global chat_history_engine
|
514 |
+
global result_metadata
|
515 |
+
kg_data = []
|
516 |
+
chat_history_engine = []
|
517 |
+
result_metadata = ""
|
518 |
+
chat_history = [[None, None]]
|
519 |
+
return chat_history
|
520 |
+
|
521 |
+
def summarize_assistant_messages(chat_history: List[ChatMessage]) -> List[ChatMessage]:
|
522 |
+
# Encontrar la anteúltima respuesta del asistente
|
523 |
+
assistant_messages = [msg for msg in chat_history if msg.role == MessageRole.ASSISTANT]
|
524 |
+
if len(assistant_messages) < 2:
|
525 |
+
return chat_history # No hay suficientes mensajes del asistente para resumir
|
526 |
+
|
527 |
+
anteultima_respuesta = assistant_messages[-2]
|
528 |
+
|
529 |
+
# Usar GPT-3.5 para generar un resumen de la anteúltima respuesta del asistente
|
530 |
+
prompt = Prompt(f"Responder SOLO con un resumen del siguiente texto: \n\n{anteultima_respuesta.content}")
|
531 |
+
response = llm.predict(prompt)
|
532 |
+
|
533 |
+
# Crear un nuevo ChatMessage con el resumen como contenido y el rol de asistente
|
534 |
+
summarized_message = ChatMessage(content=response, role=MessageRole.ASSISTANT)
|
535 |
+
|
536 |
+
# Reconstruir el historial de chat reemplazando la anteúltima respuesta del asistente con el resumen
|
537 |
+
new_chat_history = [msg if msg != anteultima_respuesta else summarized_message for msg in chat_history]
|
538 |
+
|
539 |
+
return new_chat_history
|
540 |
+
|
541 |
+
|
542 |
+
def respond(message, chat_history):
|
543 |
+
global chat_history_engine
|
544 |
+
global result_metadata
|
545 |
+
global result_texts
|
546 |
+
global final_response
|
547 |
+
global query
|
548 |
+
|
549 |
+
profile = "no-ingresado"
|
550 |
+
|
551 |
+
if profile is not None:
|
552 |
+
# Inicializar el historial de chat si está vacío con el mensaje del usuario actual
|
553 |
+
if not chat_history:
|
554 |
+
chat_history = [[message, ""]]
|
555 |
+
else:
|
556 |
+
# Agregar el mensaje actual al historial de chat
|
557 |
+
chat_history.append([message, ""])
|
558 |
+
|
559 |
+
# Resumir los mensajes previos en chat_history_engine
|
560 |
+
chat_history_engine = summarize_assistant_messages(chat_history_engine)
|
561 |
+
|
562 |
+
# Generar la respuesta usando el motor de chat
|
563 |
+
response = chat_engine.stream_chat(message, chat_history=chat_history_engine)
|
564 |
+
|
565 |
+
# Extraer la información de los metadatos y textos de la respuesta
|
566 |
+
metadata_info = extraer_informacion_metadata(response, max_results=10)
|
567 |
+
texts_info = extraer_textos_metadata(response, max_results=10)
|
568 |
+
|
569 |
+
if metadata_info:
|
570 |
+
result_metadata = "\n".join(metadata_info)
|
571 |
+
if texts_info:
|
572 |
+
result_texts = texts_info
|
573 |
+
|
574 |
+
# Procesar la respuesta generada y agregarla al historial del chat
|
575 |
+
for text in response.response_gen:
|
576 |
+
chat_history[-1][1] += text
|
577 |
+
yield "", chat_history # Actualiza el historial del chat y retorna el nuevo estado
|
578 |
+
|
579 |
+
# Guardar la conversación en el dataset
|
580 |
+
save_to_dataset(message, chat_history[-1][1], profile.name)
|
581 |
+
|
582 |
+
final_response = response
|
583 |
+
query = message
|
584 |
+
else:
|
585 |
+
simulated_message = "Ingrese con su usuario para usar el chat"
|
586 |
+
simulated_chat_history = [[message, simulated_message]] # Se genera un historial simulado
|
587 |
+
yield "", simulated_chat_history
|
588 |
+
|
589 |
+
|
590 |
+
gr.Markdown("""
|
591 |
+
# PharmaWise GMP Colombia Chat 4.7
|
592 |
+
Realiza preguntas a tus datos y obtén al final del texto las paginas y documentos utilizados generar tu responder.
|
593 |
+
""")
|
594 |
+
with gr.Row():
|
595 |
+
with gr.Column():
|
596 |
+
chatbot = gr.Chatbot(show_label=False, show_copy_button=True, ) #layout="panel"
|
597 |
+
pregunta = gr.Textbox(show_label=False, autofocus=True, placeholder="Realiza tu consulta...")
|
598 |
+
pregunta.submit(respond, [pregunta, chatbot], [pregunta, chatbot])
|
599 |
+
|
600 |
+
with gr.Row():
|
601 |
+
btn_send = gr.Button(value="Preguntar", variant="primary")
|
602 |
+
clear = gr.Button(value="Limpiar")
|
603 |
+
|
604 |
+
with gr.Row(elem_classes="examples"):
|
605 |
+
gr.Examples(label="Ejemplos", examples=["Implementación de la res. 3157 de 2018"], inputs=[pregunta])
|
606 |
+
|
607 |
+
with gr.Column():
|
608 |
+
with gr.Accordion(elem_classes="accordion", label="Bases de datos del conocimiento", open=False):
|
609 |
+
gr.Markdown("""
|
610 |
+
###### [1] Guía BPL
|
611 |
+
###### [2] Guía BPM Medicamentos
|
612 |
+
###### [3] MANUAL DE NORMAS TÉCNICAS DE CALIDAD
|
613 |
+
###### [4] Resolución 1160 - GMP - 2016
|
614 |
+
###### [5] Resolución 2266 - DECRETO FITOTERAPÉUTICOS - 2004
|
615 |
+
###### [6] Resolución 3619 - GLP - 2013
|
616 |
+
###### [7] Resolución 005107 - FITOTERAPEUTICOS - 2005
|
617 |
+
###### [8] Resolución 3690 - 2016
|
618 |
+
###### [9] Resolución 3157 - 2018
|
619 |
+
###### [10] Preguntas y respuestas Res. 3157 - 2018
|
620 |
+
""")
|
621 |
+
|
622 |
+
with gr.Accordion(elem_classes="accordion", label="Referencias", open=True):
|
623 |
+
mkdn = gr.Markdown()
|
624 |
+
|
625 |
+
with gr.Row():
|
626 |
+
btn_graph = gr.Button(value="Grafo")
|
627 |
+
btn_ref = gr.Button(value="Referencias")
|
628 |
+
btn_eval = gr.Button(value="Evaluar")
|
629 |
+
|
630 |
+
eval_accord = gr.Accordion(elem_classes="accordion", label="Evaluaciones", open=False)
|
631 |
+
with eval_accord:
|
632 |
+
evals = gr.HTML()
|
633 |
+
gr.Markdown("""| **Evaluador** | **Qué mide** | **Ejemplo de uso** | **Diferencias clave** |
|
634 |
+
|-----------------------|-------------------------------------------------------|-------------------------------------------------------|--------------------------------------------------------|
|
635 |
+
| **Groundedness** | Qué tan fundamentada está la respuesta en el contexto. | ¿La respuesta está respaldada por el contexto proporcionado? | Se enfoca en la relación entre la respuesta y el contexto. |
|
636 |
+
| **Answer Relevance** | Qué tan relevante es la respuesta para la consulta. | ¿La respuesta es pertinente a lo que el usuario preguntó? | Se centra en la relevancia de la respuesta ante la consulta. |
|
637 |
+
| **Context Relevance** | Qué tan relevante es el contexto recuperado para la consulta. | ¿El contexto obtenido es relevante para la consulta del usuario? | Se enfoca en la pertinencia del contexto en relación con la consulta. |
|
638 |
+
""")
|
639 |
+
|
640 |
+
with gr.Row():
|
641 |
+
grafo = gr.Image(label="Grafo", show_share_button=False)
|
642 |
+
|
643 |
+
with gr.Accordion(elem_classes="accordion", label="Audit trail", open=False):
|
644 |
+
with gr.Row():
|
645 |
+
with gr.Column():
|
646 |
+
available_months = load_available_logs()
|
647 |
+
default_month = available_months[-1] if available_months else None
|
648 |
+
dropdown = gr.Dropdown(choices=available_months, label="Seleccionar mes", value=default_month)
|
649 |
+
btn_logs = gr.Button(value="Actualizar")
|
650 |
+
with gr.Column():
|
651 |
+
gr.Markdown()
|
652 |
+
with gr.Column():
|
653 |
+
gr.Markdown()
|
654 |
+
# Define un DataFrame con un ancho fijo para response_text
|
655 |
+
logs_df = gr.DataFrame(headers=["Marca de Tiempo", "Mensaje Usuario", "Respuesta", "Etiqueta", "Usuario"], wrap=True, line_breaks=True)
|
656 |
+
|
657 |
+
|
658 |
+
with gr.Accordion(elem_classes="accordion", label="Referencias ampliadas", open=False):
|
659 |
+
texts = gr.HTML()
|
660 |
+
|
661 |
+
|
662 |
+
demo.load(respond, inputs=None, outputs=None)
|
663 |
+
|
664 |
+
btn_logs.click(fn=get_logs, inputs=[dropdown], outputs=[logs_df])
|
665 |
+
btn_ref.click(fn=get_ref, outputs=[mkdn, texts])
|
666 |
+
btn_eval.click(fn=get_evals, outputs=[evals, eval_accord])
|
667 |
+
btn_send.click(respond, [pregunta, chatbot], [pregunta, chatbot])
|
668 |
+
btn_graph.click(draw_graph, outputs=[grafo])
|
669 |
+
|
670 |
+
clear.click(refresh, inputs=[chatbot], outputs=[chatbot])
|
671 |
+
chatbot.like(print_like_dislike, None, None)
|
672 |
+
|
673 |
+
demo.queue(default_concurrency_limit=20)
|
674 |
+
demo.launch()
|