Upload 17 files

result/Classification/scores_amazon_counterfactual_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.7991935990685706,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.907725321888412,
+                "macro_f1": 0.6109654998349738
+            },
+            "logreg": {
+                "accuracy": 0.9206008583690987,
+                "macro_f1": 0.7553180921566123
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.930406852248394,
+                "macro_f1": 0.7991935990685706
+            }
+        }
+    }
+}

result/Classification/scores_amazon_review_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.556129066893332,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.4216,
+                "macro_f1": 0.4121755235351596
+            },
+            "logreg": {
+                "accuracy": 0.5632,
+                "macro_f1": 0.5591904800064202
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.56,
+                "macro_f1": 0.556129066893332
+            }
+        }
+    }
+}

result/Classification/scores_massive_intent_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.8148895285345188,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.7875061485489424,
+                "macro_f1": 0.735126286885438
+            },
+            "logreg": {
+                "accuracy": 0.8573536645351697,
+                "macro_f1": 0.838740737492276
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.8362474781439139,
+                "macro_f1": 0.8148895285345188
+            }
+        }
+    }
+}

result/Classification/scores_massive_scenario_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.8787774569382543,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.8691588785046729,
+                "macro_f1": 0.8610209884671544
+            },
+            "logreg": {
+                "accuracy": 0.8986719134284309,
+                "macro_f1": 0.8915311881559682
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.8823133826496301,
+                "macro_f1": 0.8787774569382543
+            }
+        }
+    }
+}

result/Clustering/scores_livedoor_news.json

@@ -0,0 +1,36 @@
+{
+    "metric_name": "v_measure_score",
+    "metric_value": 0.5096442244018489,
+    "details": {
+        "optimal_clustering_model_name": "AgglomerativeClustering",
+        "val_scores": {
+            "MiniBatchKMeans": {
+                "v_measure_score": 0.5155149790434675,
+                "homogeneity_score": 0.5040878200105626,
+                "completeness_score": 0.5274722391130744
+            },
+            "AgglomerativeClustering": {
+                "v_measure_score": 0.5484270601867571,
+                "homogeneity_score": 0.5369394189236333,
+                "completeness_score": 0.5604169964554391
+            },
+            "BisectingKMeans": {
+                "v_measure_score": 0.5376784533072557,
+                "homogeneity_score": 0.5361001139672913,
+                "completeness_score": 0.5392661137083645
+            },
+            "Birch": {
+                "v_measure_score": 0.5458063802733484,
+                "homogeneity_score": 0.5349834585205275,
+                "completeness_score": 0.5570762476737373
+            }
+        },
+        "test_scores": {
+            "AgglomerativeClustering": {
+                "v_measure_score": 0.5096442244018489,
+                "homogeneity_score": 0.5079528634930609,
+                "completeness_score": 0.5113468865915328
+            }
+        }
+    }
+}

result/Clustering/scores_mewsc16.json

@@ -0,0 +1,36 @@
+{
+    "metric_name": "v_measure_score",
+    "metric_value": 0.5141045788711239,
+    "details": {
+        "optimal_clustering_model_name": "MiniBatchKMeans",
+        "val_scores": {
+            "MiniBatchKMeans": {
+                "v_measure_score": 0.543899400881209,
+                "homogeneity_score": 0.5955190874170961,
+                "completeness_score": 0.5005147085855427
+            },
+            "AgglomerativeClustering": {
+                "v_measure_score": 0.5163023738694377,
+                "homogeneity_score": 0.5635562478103336,
+                "completeness_score": 0.47635986731721597
+            },
+            "BisectingKMeans": {
+                "v_measure_score": 0.48694697274293186,
+                "homogeneity_score": 0.5307503463138618,
+                "completeness_score": 0.4498226457773539
+            },
+            "Birch": {
+                "v_measure_score": 0.5163023738694377,
+                "homogeneity_score": 0.5635562478103336,
+                "completeness_score": 0.476359867317216
+            }
+        },
+        "test_scores": {
+            "MiniBatchKMeans": {
+                "v_measure_score": 0.5141045788711239,
+                "homogeneity_score": 0.5421348978484514,
+                "completeness_score": 0.48883029887612817
+            }
+        }
+    }
+}

result/PairClassification/scores_paws_x_ja.json

@@ -0,0 +1,41 @@
+{
+    "metric_name": "binary_f1",
+    "metric_value": 0.6211267605633802,
+    "details": {
+        "optimal_distance_metric": "euclidean_distances",
+        "val_scores": {
+            "cosine_distances": {
+                "accuracy": 0.5725,
+                "accuracy_threshold": 0.6759790182113647,
+                "binary_f1": 0.5979670522257273,
+                "binary_f1_threshold": 1.0
+            },
+            "manhatten_distances": {
+                "accuracy": 0.5985,
+                "accuracy_threshold": 30.971534729003906,
+                "binary_f1": 0.6019760056457304,
+                "binary_f1_threshold": 342.1531982421875
+            },
+            "euclidean_distances": {
+                "accuracy": 0.598,
+                "accuracy_threshold": 1.3944740295410156,
+                "binary_f1": 0.6020444131124427,
+                "binary_f1_threshold": 16.84113311767578
+            },
+            "dot_similarities": {
+                "accuracy": 0.573,
+                "accuracy_threshold": 548.4661254882812,
+                "binary_f1": 0.6016949152542372,
+                "binary_f1_threshold": 415.54345703125
+            }
+        },
+        "test_scores": {
+            "euclidean_distances": {
+                "accuracy": 0.5645,
+                "accuracy_threshold": 1.3944740295410156,
+                "binary_f1": 0.6211267605633802,
+                "binary_f1_threshold": 16.84113311767578
+            }
+        }
+    }
+}

result/Reranking/scores_esci.json

@@ -0,0 +1,31 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.9300177985352138,
+    "details": {
+        "optimal_distance_metric": "euclidean_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "ndcg@10": 0.944144892024604,
+                "ndcg@20": 0.9566345571684772,
+                "ndcg@40": 0.9640441285073341
+            },
+            "dot_score": {
+                "ndcg@10": 0.9297352149986512,
+                "ndcg@20": 0.9448677352764289,
+                "ndcg@40": 0.9537077347945042
+            },
+            "euclidean_distance": {
+                "ndcg@10": 0.9446577914383755,
+                "ndcg@20": 0.9569052614162316,
+                "ndcg@40": 0.9643388177037328
+            }
+        },
+        "test_scores": {
+            "euclidean_distance": {
+                "ndcg@10": 0.9300177985352138,
+                "ndcg@20": 0.9471226984711035,
+                "ndcg@40": 0.9564853495563737
+            }
+        }
+    }
+}

result/Retrieval/scores_jagovfaqs_22k.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.736494039429321,
+    "details": {
+        "optimal_distance_metric": "euclidean_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.584381398069611,
+                "accuracy@3": 0.766890903773033,
+                "accuracy@5": 0.8215852588476162,
+                "accuracy@10": 0.8748171980111143,
+                "ndcg@10": 0.7326724272945226,
+                "mrr@10": 0.6867933397401084
+            },
+            "dot_score": {
+                "accuracy@1": 0.3275811640830652,
+                "accuracy@3": 0.49575899385785316,
+                "accuracy@5": 0.5560105293945599,
+                "accuracy@10": 0.6326411231354198,
+                "ndcg@10": 0.47570376078893195,
+                "mrr@10": 0.4260534037150003
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.5867212635273472,
+                "accuracy@3": 0.7712781515062884,
+                "accuracy@5": 0.8248025738520035,
+                "accuracy@10": 0.870722433460076,
+                "ndcg@10": 0.733274373935001,
+                "mrr@10": 0.688727094620631
+            }
+        },
+        "test_scores": {
+            "euclidean_distance": {
+                "accuracy@1": 0.5929824561403508,
+                "accuracy@3": 0.7713450292397661,
+                "accuracy@5": 0.8190058479532164,
+                "accuracy@10": 0.8730994152046784,
+                "ndcg@10": 0.736494039429321,
+                "mrr@10": 0.6924176181193723
+            }
+        }
+    }
+}

result/Retrieval/scores_jaqket.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.484437639428696,
+    "details": {
+        "optimal_distance_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.32964824120603015,
+                "accuracy@3": 0.49246231155778897,
+                "accuracy@5": 0.5718592964824121,
+                "accuracy@10": 0.6522613065326633,
+                "ndcg@10": 0.48355974964323734,
+                "mrr@10": 0.4303046980936424
+            },
+            "dot_score": {
+                "accuracy@1": 0.1306532663316583,
+                "accuracy@3": 0.24824120603015076,
+                "accuracy@5": 0.3005025125628141,
+                "accuracy@10": 0.4120603015075377,
+                "ndcg@10": 0.2557712936565093,
+                "mrr@10": 0.20779014118210082
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.32663316582914576,
+                "accuracy@3": 0.4864321608040201,
+                "accuracy@5": 0.5618090452261306,
+                "accuracy@10": 0.6462311557788945,
+                "ndcg@10": 0.47854687532079854,
+                "mrr@10": 0.4257059105049051
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.31995987963891676,
+                "accuracy@3": 0.4954864593781344,
+                "accuracy@5": 0.5777331995987964,
+                "accuracy@10": 0.6619859578736209,
+                "ndcg@10": 0.484437639428696,
+                "mrr@10": 0.4284834662718312
+            }
+        }
+    }
+}

result/Retrieval/scores_mrtydi.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.3342716158897666,
+    "details": {
+        "optimal_distance_metric": "euclidean_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.21551724137931033,
+                "accuracy@3": 0.3609913793103448,
+                "accuracy@5": 0.42995689655172414,
+                "accuracy@10": 0.509698275862069,
+                "ndcg@10": 0.3532634556091773,
+                "mrr@10": 0.30418462643678146
+            },
+            "dot_score": {
+                "accuracy@1": 0.04310344827586207,
+                "accuracy@3": 0.08836206896551724,
+                "accuracy@5": 0.11530172413793104,
+                "accuracy@10": 0.15301724137931033,
+                "ndcg@10": 0.09304385842795544,
+                "mrr@10": 0.0744680487137384
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.2252155172413793,
+                "accuracy@3": 0.3760775862068966,
+                "accuracy@5": 0.4418103448275862,
+                "accuracy@10": 0.53125,
+                "ndcg@10": 0.3684043180071825,
+                "mrr@10": 0.31742739121510655
+            }
+        },
+        "test_scores": {
+            "euclidean_distance": {
+                "accuracy@1": 0.2013888888888889,
+                "accuracy@3": 0.375,
+                "accuracy@5": 0.4638888888888889,
+                "accuracy@10": 0.5541666666666667,
+                "ndcg@10": 0.3342716158897666,
+                "mrr@10": 0.30838458994708995
+            }
+        }
+    }
+}

result/Retrieval/scores_nlp_journal_abs_intro.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.8768878489670099,
+    "details": {
+        "optimal_distance_metric": "euclidean_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.82,
+                "accuracy@3": 0.94,
+                "accuracy@5": 0.96,
+                "accuracy@10": 0.96,
+                "ndcg@10": 0.8990879114471843,
+                "mrr@10": 0.8783333333333333
+            },
+            "dot_score": {
+                "accuracy@1": 0.68,
+                "accuracy@3": 0.81,
+                "accuracy@5": 0.88,
+                "accuracy@10": 0.96,
+                "ndcg@10": 0.8092872591620544,
+                "mrr@10": 0.7623293650793651
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.85,
+                "accuracy@3": 0.94,
+                "accuracy@5": 0.96,
+                "accuracy@10": 0.97,
+                "ndcg@10": 0.9113031320591165,
+                "mrr@10": 0.8921666666666667
+            }
+        },
+        "test_scores": {
+            "euclidean_distance": {
+                "accuracy@1": 0.801980198019802,
+                "accuracy@3": 0.8935643564356436,
+                "accuracy@5": 0.9331683168316832,
+                "accuracy@10": 0.948019801980198,
+                "ndcg@10": 0.8768878489670099,
+                "mrr@10": 0.8536637592330663
+            }
+        }
+    }
+}

result/Retrieval/scores_nlp_journal_title_abs.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.9716879343439146,
+    "details": {
+        "optimal_distance_metric": "euclidean_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.9,
+                "accuracy@3": 0.96,
+                "accuracy@5": 0.97,
+                "accuracy@10": 1.0,
+                "ndcg@10": 0.9491865865926399,
+                "mrr@10": 0.933095238095238
+            },
+            "dot_score": {
+                "accuracy@1": 0.68,
+                "accuracy@3": 0.86,
+                "accuracy@5": 0.92,
+                "accuracy@10": 0.92,
+                "ndcg@10": 0.8106756888962013,
+                "mrr@10": 0.774
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.9,
+                "accuracy@3": 0.96,
+                "accuracy@5": 0.97,
+                "accuracy@10": 1.0,
+                "ndcg@10": 0.9493652711581159,
+                "mrr@10": 0.9332738095238095
+            }
+        },
+        "test_scores": {
+            "euclidean_distance": {
+                "accuracy@1": 0.9381188118811881,
+                "accuracy@3": 0.9826732673267327,
+                "accuracy@5": 0.9925742574257426,
+                "accuracy@10": 0.9975247524752475,
+                "ndcg@10": 0.9716879343439146,
+                "mrr@10": 0.9630225522552256
+            }
+        }
+    }
+}

result/Retrieval/scores_nlp_journal_title_intro.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.7608660955794895,
+    "details": {
+        "optimal_distance_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.63,
+                "accuracy@3": 0.8,
+                "accuracy@5": 0.85,
+                "accuracy@10": 0.89,
+                "ndcg@10": 0.763548354665835,
+                "mrr@10": 0.7226944444444446
+            },
+            "dot_score": {
+                "accuracy@1": 0.39,
+                "accuracy@3": 0.64,
+                "accuracy@5": 0.7,
+                "accuracy@10": 0.83,
+                "ndcg@10": 0.6048238230337044,
+                "mrr@10": 0.5335158730158731
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.63,
+                "accuracy@3": 0.8,
+                "accuracy@5": 0.85,
+                "accuracy@10": 0.88,
+                "ndcg@10": 0.7605380547091953,
+                "mrr@10": 0.7215833333333335
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.655940594059406,
+                "accuracy@3": 0.7821782178217822,
+                "accuracy@5": 0.8341584158415841,
+                "accuracy@10": 0.8638613861386139,
+                "ndcg@10": 0.7608660955794895,
+                "mrr@10": 0.7275842762847713
+            }
+        }
+    }
+}

result/STS/scores_jsick.json

@@ -0,0 +1,31 @@
+{
+    "metric_name": "spearman",
+    "metric_value": 0.8343927017558587,
+    "details": {
+        "optimal_similarity_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "pearson": 0.8446183563656585,
+                "spearman": 0.8367170844896946
+            },
+            "manhatten_distance": {
+                "pearson": 0.8472993028173297,
+                "spearman": 0.8343797412370403
+            },
+            "euclidean_distance": {
+                "pearson": 0.8472993028173297,
+                "spearman": 0.8343797412370403
+            },
+            "dot_score": {
+                "pearson": 0.8211250471651956,
+                "spearman": 0.8029357210718526
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "pearson": 0.8400443214276277,
+                "spearman": 0.8343927017558587
+            }
+        }
+    }
+}

result/STS/scores_jsts.json

@@ -0,0 +1,31 @@
+{
+    "metric_name": "spearman",
+    "metric_value": 0.8213297790184827,
+    "details": {
+        "optimal_similarity_metric": "manhatten_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "pearson": 0.8341982404976528,
+                "spearman": 0.7825049368314301
+            },
+            "manhatten_distance": {
+                "pearson": 0.8221562123952046,
+                "spearman": 0.7830096343603999
+            },
+            "euclidean_distance": {
+                "pearson": 0.8221562123952046,
+                "spearman": 0.7830096343603999
+            },
+            "dot_score": {
+                "pearson": 0.8056563438618474,
+                "spearman": 0.7471043843298736
+            }
+        },
+        "test_scores": {
+            "manhatten_distance": {
+                "pearson": 0.8556382223713024,
+                "spearman": 0.8213297790184827
+            }
+        }
+    }
+}

result/summary.json

@@ -0,0 +1,62 @@
+{
+    "Classification": {
+        "amazon_counterfactual_classification": {
+            "macro_f1": 0.7991935990685706
+        },
+        "amazon_review_classification": {
+            "macro_f1": 0.556129066893332
+        },
+        "massive_intent_classification": {
+            "macro_f1": 0.8148895285345188
+        },
+        "massive_scenario_classification": {
+            "macro_f1": 0.8787774569382543
+        }
+    },
+    "Reranking": {
+        "esci": {
+            "ndcg@10": 0.9300177985352138
+        }
+    },
+    "Retrieval": {
+        "jagovfaqs_22k": {
+            "ndcg@10": 0.736494039429321
+        },
+        "jaqket": {
+            "ndcg@10": 0.484437639428696
+        },
+        "mrtydi": {
+            "ndcg@10": 0.3342716158897666
+        },
+        "nlp_journal_abs_intro": {
+            "ndcg@10": 0.8768878489670099
+        },
+        "nlp_journal_title_abs": {
+            "ndcg@10": 0.9716879343439146
+        },
+        "nlp_journal_title_intro": {
+            "ndcg@10": 0.7608660955794895
+        }
+    },
+    "STS": {
+        "jsick": {
+            "spearman": 0.8343927017558587
+        },
+        "jsts": {
+            "spearman": 0.8213297790184827
+        }
+    },
+    "Clustering": {
+        "livedoor_news": {
+            "v_measure_score": 0.5096442244018489
+        },
+        "mewsc16": {
+            "v_measure_score": 0.5141045788711239
+        }
+    },
+    "PairClassification": {
+        "paws_x_ja": {
+            "binary_f1": 0.6211267605633802
+        }
+    }
+}