Upload benchmark_v2/all_results.json with huggingface_hub
Browse files
benchmark_v2/all_results.json
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"task1_cve_linkage": {
|
| 3 |
+
"BM25": {
|
| 4 |
+
"R@1": 0.445,
|
| 5 |
+
"R@5": 0.64,
|
| 6 |
+
"R@10": 0.707,
|
| 7 |
+
"MRR": 0.528,
|
| 8 |
+
"verified": true
|
| 9 |
+
},
|
| 10 |
+
"all-MiniLM-L6-v2": {
|
| 11 |
+
"R@1": 0.431,
|
| 12 |
+
"R@5": 0.656,
|
| 13 |
+
"R@10": 0.727,
|
| 14 |
+
"MRR": 0.53,
|
| 15 |
+
"verified": true
|
| 16 |
+
},
|
| 17 |
+
"all-mpnet-base-v2": {
|
| 18 |
+
"R@1": 0.46,
|
| 19 |
+
"R@5": 0.64,
|
| 20 |
+
"R@10": 0.718,
|
| 21 |
+
"MRR": 0.54,
|
| 22 |
+
"verified": true
|
| 23 |
+
},
|
| 24 |
+
"Hybrid BM25+mpnet": {
|
| 25 |
+
"R@1": 0.486,
|
| 26 |
+
"R@5": 0.675,
|
| 27 |
+
"R@10": 0.736,
|
| 28 |
+
"MRR": 0.567,
|
| 29 |
+
"verified": true
|
| 30 |
+
}
|
| 31 |
+
},
|
| 32 |
+
"task2_signal_detection": {
|
| 33 |
+
"Majority": {
|
| 34 |
+
"F1": 0.0,
|
| 35 |
+
"Precision": 0.0,
|
| 36 |
+
"Recall": 0.0,
|
| 37 |
+
"verified": true
|
| 38 |
+
},
|
| 39 |
+
"TF-IDF + LR": {
|
| 40 |
+
"F1": 0.928,
|
| 41 |
+
"Precision": 1.0,
|
| 42 |
+
"Recall": 0.865,
|
| 43 |
+
"verified": true
|
| 44 |
+
},
|
| 45 |
+
"SecBERT (FT)": {
|
| 46 |
+
"F1": 0.943,
|
| 47 |
+
"Precision": 1.0,
|
| 48 |
+
"Recall": 0.892,
|
| 49 |
+
"verified": true
|
| 50 |
+
}
|
| 51 |
+
},
|
| 52 |
+
"task3_temporal_generalization": {
|
| 53 |
+
"BM25": {
|
| 54 |
+
"R@1": 0.443,
|
| 55 |
+
"R@5": 0.654,
|
| 56 |
+
"R@10": 0.715,
|
| 57 |
+
"MRR": 0.533,
|
| 58 |
+
"verified": true
|
| 59 |
+
},
|
| 60 |
+
"all-MiniLM-L6-v2": {
|
| 61 |
+
"R@1": 0.453,
|
| 62 |
+
"R@5": 0.669,
|
| 63 |
+
"R@10": 0.728,
|
| 64 |
+
"MRR": 0.546,
|
| 65 |
+
"verified": true
|
| 66 |
+
},
|
| 67 |
+
"all-mpnet-base-v2": {
|
| 68 |
+
"R@1": 0.461,
|
| 69 |
+
"R@5": 0.657,
|
| 70 |
+
"R@10": 0.725,
|
| 71 |
+
"MRR": 0.544,
|
| 72 |
+
"verified": true
|
| 73 |
+
},
|
| 74 |
+
"Hybrid BM25+mpnet": {
|
| 75 |
+
"R@1": 0.491,
|
| 76 |
+
"R@5": 0.69,
|
| 77 |
+
"R@10": 0.743,
|
| 78 |
+
"MRR": 0.577,
|
| 79 |
+
"verified": true
|
| 80 |
+
}
|
| 81 |
+
},
|
| 82 |
+
"meta": {
|
| 83 |
+
"corpus_size": 340536,
|
| 84 |
+
"task1_test_size": 550,
|
| 85 |
+
"task2_test_size": 407,
|
| 86 |
+
"task3_test_size": 393,
|
| 87 |
+
"hardware": "macOS arm64 (Apple MPS)",
|
| 88 |
+
"python_version": "3.12",
|
| 89 |
+
"pytorch_version": "2.4"
|
| 90 |
+
}
|
| 91 |
+
}
|