geekyrakshit's picture
add: limited eval table rendering in app
785c044
raw
history blame
1.98 kB
import os
import pandas as pd
import pymupdf4llm
import weave
import weave.trace
from firerequests import FireRequests
@weave.op()
def get_markdown_from_pdf_url(url: str) -> str:
FireRequests().download(url, "temp.pdf", show_progress=False)
markdown = pymupdf4llm.to_markdown("temp.pdf", show_progress=False)
os.remove("temp.pdf")
return markdown
class EvaluationCallManager:
def __init__(self, entity: str, project: str, call_id: str, max_count: int = 10):
self.base_call = weave.init(f"{entity}/{project}").get_call(call_id=call_id)
self.max_count = max_count
self.show_warning_in_app = False
self.call_list = []
def collect_guardrail_guard_calls_from_eval(self, call):
guard_calls, count = [], 0
for eval_predict_call in call.children():
if "Evaluation.summarize" in eval_predict_call._op_name:
break
required_call = eval_predict_call.children()[0].children()[0].children()[0]
guard_calls.append(
{
"input_prompt": str(required_call.inputs["prompt"]),
"outputs": dict(required_call.output),
}
)
count += 1
if count >= self.max_count:
self.show_warning_in_app = True
break
return guard_calls
def render_calls_to_streamlit(self):
dataframe = {
"input_prompt": [
call["input_prompt"] for call in self.call_list[0]["calls"]
]
}
for guardrail_call in self.call_list:
dataframe[guardrail_call["guardrail_name"] + ".safe"] = [
call["outputs"]["safe"] for call in guardrail_call["calls"]
]
dataframe[guardrail_call["guardrail_name"] + ".summary"] = [
call["outputs"]["summary"] for call in guardrail_call["calls"]
]
return pd.DataFrame(dataframe)