add first Tower

evals/arc/arc_nl_TowerBase-7B-v0.1.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_nl": {
+      "acc": 0.358426005132592,
+      "acc_stderr": 0.01403142278327522,
+      "acc_norm": 0.3721129170230967,
+      "acc_norm_stderr": 0.014143494499252762
+    }
+  },
+  "versions": {
+    "arc_nl": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=Unbabel/TowerBase-7B-v0.1,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 4,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc/arc_nl_TowerInstruct-7B-v0.1.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_nl": {
+      "acc": 0.3900769888793841,
+      "acc_stderr": 0.014272210869539614,
+      "acc_norm": 0.3943541488451668,
+      "acc_norm_stderr": 0.014299839367505047
+    }
+  },
+  "versions": {
+    "arc_nl": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=Unbabel/TowerInstruct-7B-v0.1,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 4,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/hellaswag/hellaswag_nl_TowerBase-7B-v0.1.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "hellaswag_nl": {
+      "acc": 0.4561252023745278,
+      "acc_stderr": 0.005174784023983772,
+      "acc_norm": 0.5817593092282785,
+      "acc_norm_stderr": 0.005124901694100509
+    }
+  },
+  "versions": {
+    "hellaswag_nl": 1
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=Unbabel/TowerBase-7B-v0.1,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 4,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/hellaswag/hellaswag_nl_TowerInstruct-7B-v0.1.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "hellaswag_nl": {
+      "acc": 0.471127900701565,
+      "acc_stderr": 0.005186154697710561,
+      "acc_norm": 0.5960064759848893,
+      "acc_norm_stderr": 0.005098159491383791
+    }
+  },
+  "versions": {
+    "hellaswag_nl": 1
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=Unbabel/TowerInstruct-7B-v0.1,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 4,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/models.json

@@ -143,6 +143,22 @@
         "num_parameters": 2779683840,
         "quantization": "8-bit"
     },
+    "towerbase-7b-v0.1": {
+        "compute_dtype": "auto",
+        "dutch_coverage": "not-given",
+        "model_name": "Unbabel/TowerBase-7B-v0.1",
+        "model_type": "not-given",
+        "num_parameters": 6738415616,
+        "quantization": "8-bit"
+    },
+    "towerinstruct-7b-v0.1": {
+        "compute_dtype": "auto",
+        "dutch_coverage": "not-given",
+        "model_name": "Unbabel/TowerInstruct-7B-v0.1",
+        "model_type": "not-given",
+        "num_parameters": 6738472960,
+        "quantization": "8-bit"
+    },
     "yi-6b": {
         "compute_dtype": "auto",
         "dutch_coverage": "none",

evals/truthfulqa/truthfulqa_nl_TowerBase-7B-v0.1.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "truthfulqa_nl": {
+      "mc1": 0.25987261146496815,
+      "mc1_stderr": 0.015663018533664023,
+      "mc2": 0.409372470832061,
+      "mc2_stderr": 0.014507968477947334
+    }
+  },
+  "versions": {
+    "truthfulqa_nl": 1
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=Unbabel/TowerBase-7B-v0.1,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 4,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/truthfulqa/truthfulqa_nl_TowerInstruct-7B-v0.1.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "truthfulqa_nl": {
+      "mc1": 0.30318471337579617,
+      "mc1_stderr": 0.01641551278171108,
+      "mc2": 0.4474541407756634,
+      "mc2_stderr": 0.01525070581914585
+    }
+  },
+  "versions": {
+    "truthfulqa_nl": 1
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=Unbabel/TowerInstruct-7B-v0.1,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 4,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}