add evals

evals/arc/arc_nl_Yi-6B.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_nl": {
+      "acc": 0.2711719418306245,
+      "acc_stderr": 0.01300808888870032,
+      "acc_norm": 0.2951240376390077,
+      "acc_norm_stderr": 0.013345572865502647
+    }
+  },
+  "versions": {
+    "arc_nl": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=01-ai/Yi-6B,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 2,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/hellaswag/hellaswag_nl_Yi-6B.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "hellaswag_nl": {
+      "acc": 0.34225580140313006,
+      "acc_stderr": 0.004929520183215306,
+      "acc_norm": 0.41640582838640045,
+      "acc_norm_stderr": 0.005121705462751119
+    }
+  },
+  "versions": {
+    "hellaswag_nl": 1
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=01-ai/Yi-6B,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 2,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/hellaswag/hellaswag_nl_phi-2.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "hellaswag_nl": {
+      "acc": 0.2746896923907178,
+      "acc_stderr": 0.004637499043161815,
+      "acc_norm": 0.3019967620075553,
+      "acc_norm_stderr": 0.004770134104673541
+    }
+  },
+  "versions": {
+    "hellaswag_nl": 1
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=microsoft/phi-2,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 4,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/mmlu/mmlu_nl_Yi-6B-Chat.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "mmlu_nl": {
+      "acc": 0.4071488199134856,
+      "acc_stderr": 0.004280136715848525,
+      "acc_norm": 0.3481065492904303,
+      "acc_norm_stderr": 0.004150043156295438
+    }
+  },
+  "versions": {
+    "mmlu_nl": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=01-ai/Yi-6B-Chat,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 2,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/models.json

@@ -135,6 +135,22 @@
         "num_parameters": 6738440192,
         "quantization": "8-bit"
     },
+    "phi-2": {
+        "compute_dtype": "auto",
+        "dutch_coverage": "none",
+        "model_name": "microsoft/phi-2",
+        "model_type": "instruction-tuned",
+        "num_parameters": 2779683840,
+        "quantization": "8-bit"
+    },
+    "yi-6b": {
+        "compute_dtype": "auto",
+        "dutch_coverage": "none",
+        "model_name": "01-ai/Yi-6B",
+        "model_type": "pretrained",
+        "num_parameters": 6061035520,
+        "quantization": "8-bit"
+    },
     "yi-6b-chat": {
         "compute_dtype": "auto",
         "dutch_coverage": "none",

evals/truthfulqa/truthfulqa_nl_Yi-6B.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "truthfulqa_nl": {
+      "mc1": 0.2624203821656051,
+      "mc1_stderr": 0.015712496862048314,
+      "mc2": 0.4268865025462531,
+      "mc2_stderr": 0.015393899192344402
+    }
+  },
+  "versions": {
+    "truthfulqa_nl": 1
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=01-ai/Yi-6B,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 2,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}