Spaces:
Runtime error
Runtime error
import os | |
import numpy as np | |
import pandas as pd | |
from inspiredco.critique import Critique | |
from zeno import ZenoOptions, distill, metric, model | |
# client = Critique(api_key=os.environ["INSPIREDCO_API_KEY"]) | |
def pred_fns(name): | |
def pred(df, ops): | |
model_df = pd.read_csv( | |
ops.label_path + "/{}.tsv".format(name), | |
sep="\t", | |
quoting=3, | |
keep_default_na=False, | |
) | |
embed_df = pd.read_csv( | |
ops.label_path + "/ref_embed.tsv", | |
sep="\t", | |
keep_default_na=False, | |
quoting=3, | |
) | |
df_join = df[["text"]].merge( | |
model_df[["text", "translation"]], on="text", how="left" | |
) | |
df_join = df_join.merge(embed_df, on="text", how="left") | |
return df_join["translation"].fillna(""), [ | |
np.fromstring(d[1:-1], sep=",") for d in df_join["embed"] | |
] | |
return pred | |
def bert_score(df, ops): | |
eval_dict = df[["source", ops.output_column, "reference"]].to_dict("records") | |
for d in eval_dict: | |
d["references"] = [d.pop("reference")] | |
d["target"] = d.pop(ops.output_column) | |
result = client.evaluate( | |
metric="bert_score", config={"model": "bert-base-uncased"}, dataset=eval_dict | |
) | |
return [round(r["value"], 6) for r in result["examples"]] | |
def bleu(df, ops): | |
eval_dict = df[[ops.output_column, "reference"]].to_dict("records") | |
for d in eval_dict: | |
d["references"] = [d.pop("reference")] | |
d["target"] = d.pop(ops.output_column) | |
result = client.evaluate( | |
metric="bleu", | |
config={"smooth_method": "add_k", "smooth-value": 1.0}, | |
dataset=eval_dict, | |
) | |
return [round(r["value"], 6) for r in result["examples"]] | |
def chrf(df, ops): | |
eval_dict = df[[ops.output_column, "reference"]].to_dict("records") | |
for d in eval_dict: | |
d["references"] = [d.pop("reference")] | |
d["target"] = d.pop(ops.output_column) | |
result = client.evaluate( | |
metric="chrf", | |
config={}, | |
dataset=eval_dict, | |
) | |
return [round(r["value"], 6) for r in result["examples"]] | |
def length_ratio(df, ops): | |
eval_dict = df[[ops.output_column, "reference"]].to_dict("records") | |
for d in eval_dict: | |
d["references"] = [d.pop("reference")] | |
d["target"] = d.pop(ops.output_column) | |
result = client.evaluate( | |
metric="length_ratio", | |
config={}, | |
dataset=eval_dict, | |
) | |
return [round(r["value"], 6) for r in result["examples"]] | |
def avg_bert_score(df, ops: ZenoOptions): | |
return df[ops.distill_columns["bert_score"]].mean() | |
def avg_bleu(df, ops: ZenoOptions): | |
return df[ops.distill_columns["bleu"]].mean() | |
def avg_chrf(df, ops: ZenoOptions): | |
return df[ops.distill_columns["chrf"]].mean() | |
def avg_length_ratio(df, ops: ZenoOptions): | |
return df[ops.distill_columns["length_ratio"]].mean() | |
def length(df, ops): | |
return df[ops.data_column].str.len() | |