more metrics

.zeno_cache/POSTDISTILLbert_scorehuman.pickle

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b3874299934f57785c20ef632025c3caceb8cffc9b9029a3c026c7366be7fac4
+oid sha256:1e7695f9c6114d019cda0942d59536b9666acb7c77b0ab76064ac3413844d401
 size 275525

.zeno_cache/{OUTPUThuman-with-embeddings.pickle → POSTDISTILLbleuhuman.pickle}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f92db696d6c39e571601552125d0b4dd2a6382071394ce0693f71fafbdab5da
-size 280865
+oid sha256:a452a66042652393d61b1c46a83395d98d543498fb6e5825d5c5c52df57da4f3
+size 275519

.zeno_cache/{POSTDISTILLbert_scorehuman-with-embeddings.pickle → POSTDISTILLchrfhuman.pickle}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:374317448ac7ec37fa499adfa775c4370a7cf3feca2854bc861810f075a398d5
-size 25744
+oid sha256:014caba05b6980c1c71d9602952474737446c8ffee54f49f12bd4bd9b9987375
+size 275519

.zeno_cache/{EMBEDDINGhuman-with-embeddings.pickle → POSTDISTILLlength_ratiohuman.pickle}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e2ddbd958723a349787a13b792e697688b04c1b1c057137db818af26c1936c9
-size 3477209
+oid sha256:1205b1c225a4fa82c63138540230f1f03bb0f19292dd98fc22284512d437ee37
+size 275527

model.py

@@ -3,7 +3,7 @@ from inspiredco.critique import Critique
 import os
 from sentence_transformers import SentenceTransformer
 
-# client = Critique(api_key=os.environ["INSPIREDCO_API_KEY"])
+client = Critique(api_key=os.environ["INSPIREDCO_API_KEY"])
 
 
 @model
@@ -19,15 +19,62 @@ def pred_fns(name):
 
 @distill
 def bert_score(df, ops):
-    eval_dict = df[["source", ops.output_column, "label"]].to_dict("records")
+    eval_dict = df[["source", ops.output_column, "reference"]].to_dict("records")
     for d in eval_dict:
-        d["references"] = [d.pop("label")]
+        d["references"] = [d.pop("reference")]
         d["target"] = d.pop(ops.output_column)
 
-    # result = client.evaluate(
-    #     metric="bert_score", config={"model": "bert-base-uncased"}, dataset=eval_dict
-    # )
-    result = {"examples": [{"value": 0.5} for _ in range(len(eval_dict))]}
+    result = client.evaluate(
+        metric="bert_score", config={"model": "bert-base-uncased"}, dataset=eval_dict
+    )
+
+    return [round(r["value"], 6) for r in result["examples"]]
+
+
+@distill
+def bleu(df, ops):
+    eval_dict = df[[ops.output_column, "reference"]].to_dict("records")
+    for d in eval_dict:
+        d["references"] = [d.pop("reference")]
+        d["target"] = d.pop(ops.output_column)
+
+    result = client.evaluate(
+        metric="bleu",
+        config={"smooth_method": "add_k", "smooth-value": 1.0},
+        dataset=eval_dict,
+    )
+
+    return [round(r["value"], 6) for r in result["examples"]]
+
+
+@distill
+def chrf(df, ops):
+    eval_dict = df[[ops.output_column, "reference"]].to_dict("records")
+    for d in eval_dict:
+        d["references"] = [d.pop("reference")]
+        d["target"] = d.pop(ops.output_column)
+
+    result = client.evaluate(
+        metric="chrf",
+        config={},
+        dataset=eval_dict,
+    )
+
+    return [round(r["value"], 6) for r in result["examples"]]
+
+
+@distill
+def length_ratio(df, ops):
+    eval_dict = df[[ops.output_column, "reference"]].to_dict("records")
+    for d in eval_dict:
+        d["references"] = [d.pop("reference")]
+        d["target"] = d.pop(ops.output_column)
+
+    result = client.evaluate(
+        metric="length_ratio",
+        config={},
+        dataset=eval_dict,
+    )
 
     return [round(r["value"], 6) for r in result["examples"]]
 
@@ -37,6 +84,21 @@ def avg_bert_score(df, ops: ZenoOptions):
     return df[ops.distill_columns["bert_score"]].mean()
 
 
+@metric
+def avg_bleu(df, ops: ZenoOptions):
+    return df[ops.distill_columns["bleu"]].mean()
+
+
+@metric
+def avg_chrf(df, ops: ZenoOptions):
+    return df[ops.distill_columns["chrf"]].mean()
+
+
+@metric
+def avg_length_ratio(df, ops: ZenoOptions):
+    return df[ops.distill_columns["length_ratio"]].mean()
+
+
 @distill
 def length(df, ops):
     return df[ops.data_column].str.len()