Upload 17 files

result/Classification/scores_amazon_counterfactual_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.7665550732749669,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.9098712446351931,
+                "macro_f1": 0.6139035745285253
+            },
+            "logreg": {
+                "accuracy": 0.9206008583690987,
+                "macro_f1": 0.7381028328396749
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.923982869379015,
+                "macro_f1": 0.7665550732749669
+            }
+        }
+    }
+}

result/Classification/scores_amazon_review_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.5575876111411316,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.4314,
+                "macro_f1": 0.4209604852624187
+            },
+            "logreg": {
+                "accuracy": 0.5702,
+                "macro_f1": 0.5653832808449197
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.562,
+                "macro_f1": 0.5575876111411316
+            }
+        }
+    }
+}

result/Classification/scores_massive_intent_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.8141210121425055,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.7757009345794392,
+                "macro_f1": 0.7456574019302791
+            },
+            "logreg": {
+                "accuracy": 0.8421052631578947,
+                "macro_f1": 0.8271757887821682
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.8416274377942166,
+                "macro_f1": 0.8141210121425055
+            }
+        }
+    }
+}

result/Classification/scores_massive_scenario_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.8848812917656395,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.8657156910969012,
+                "macro_f1": 0.8581068338871749
+            },
+            "logreg": {
+                "accuracy": 0.8898180029513035,
+                "macro_f1": 0.887764836229313
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.8860121049092132,
+                "macro_f1": 0.8848812917656395
+            }
+        }
+    }
+}

result/Clustering/scores_livedoor_news.json

@@ -0,0 +1,36 @@
+{
+    "metric_name": "v_measure_score",
+    "metric_value": 0.5427223607801758,
+    "details": {
+        "optimal_clustering_model_name": "BisectingKMeans",
+        "val_scores": {
+            "MiniBatchKMeans": {
+                "v_measure_score": 0.5453092926343514,
+                "homogeneity_score": 0.5376167786682042,
+                "completeness_score": 0.5532251395371498
+            },
+            "AgglomerativeClustering": {
+                "v_measure_score": 0.5221218542278205,
+                "homogeneity_score": 0.5145096860981694,
+                "completeness_score": 0.5299626488611732
+            },
+            "BisectingKMeans": {
+                "v_measure_score": 0.5498693214751904,
+                "homogeneity_score": 0.5475063196854639,
+                "completeness_score": 0.552252808804315
+            },
+            "Birch": {
+                "v_measure_score": 0.5208037508658081,
+                "homogeneity_score": 0.5132767763409753,
+                "completeness_score": 0.5285547703444661
+            }
+        },
+        "test_scores": {
+            "BisectingKMeans": {
+                "v_measure_score": 0.5427223607801758,
+                "homogeneity_score": 0.5417341205522448,
+                "completeness_score": 0.5437142131253088
+            }
+        }
+    }
+}

result/Clustering/scores_mewsc16.json

@@ -0,0 +1,36 @@
+{
+    "metric_name": "v_measure_score",
+    "metric_value": 0.5404099864321413,
+    "details": {
+        "optimal_clustering_model_name": "AgglomerativeClustering",
+        "val_scores": {
+            "MiniBatchKMeans": {
+                "v_measure_score": 0.502791381026052,
+                "homogeneity_score": 0.5517784337158165,
+                "completeness_score": 0.46179324043437603
+            },
+            "AgglomerativeClustering": {
+                "v_measure_score": 0.5302546097654716,
+                "homogeneity_score": 0.5735135314580632,
+                "completeness_score": 0.4930638394517115
+            },
+            "BisectingKMeans": {
+                "v_measure_score": 0.48656257334532493,
+                "homogeneity_score": 0.5342920872487864,
+                "completeness_score": 0.4466613135580361
+            },
+            "Birch": {
+                "v_measure_score": 0.49305647750510134,
+                "homogeneity_score": 0.5374392451928177,
+                "completeness_score": 0.45544495608862656
+            }
+        },
+        "test_scores": {
+            "AgglomerativeClustering": {
+                "v_measure_score": 0.5404099864321413,
+                "homogeneity_score": 0.5789428395923124,
+                "completeness_score": 0.5066863291321174
+            }
+        }
+    }
+}

result/PairClassification/scores_paws_x_ja.json

@@ -0,0 +1,41 @@
+{
+    "metric_name": "binary_f1",
+    "metric_value": 0.6237623762376238,
+    "details": {
+        "optimal_distance_metric": "euclidean_distances",
+        "val_scores": {
+            "cosine_distances": {
+                "accuracy": 0.5725,
+                "accuracy_threshold": 0.6920696496963501,
+                "binary_f1": 0.5979670522257273,
+                "binary_f1_threshold": 1.0
+            },
+            "manhatten_distances": {
+                "accuracy": 0.6015,
+                "accuracy_threshold": 19.63576316833496,
+                "binary_f1": 0.6017636684303351,
+                "binary_f1_threshold": 274.46441650390625
+            },
+            "euclidean_distances": {
+                "accuracy": 0.602,
+                "accuracy_threshold": 0.9731899499893188,
+                "binary_f1": 0.6019760056457304,
+                "binary_f1_threshold": 12.281266212463379
+            },
+            "dot_similarities": {
+                "accuracy": 0.574,
+                "accuracy_threshold": 332.39276123046875,
+                "binary_f1": 0.6014825273561596,
+                "binary_f1_threshold": 263.39337158203125
+            }
+        },
+        "test_scores": {
+            "euclidean_distances": {
+                "accuracy": 0.566,
+                "accuracy_threshold": 0.9731899499893188,
+                "binary_f1": 0.6237623762376238,
+                "binary_f1_threshold": 12.281266212463379
+            }
+        }
+    }
+}

result/Reranking/scores_esci.json

@@ -0,0 +1,31 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.9290942178703699,
+    "details": {
+        "optimal_distance_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "ndcg@10": 0.9419326097489188,
+                "ndcg@20": 0.9546274758967366,
+                "ndcg@40": 0.9625015652058491
+            },
+            "dot_score": {
+                "ndcg@10": 0.933159692803982,
+                "ndcg@20": 0.9482607249371672,
+                "ndcg@40": 0.956621759096631
+            },
+            "euclidean_distance": {
+                "ndcg@10": 0.9418339438093611,
+                "ndcg@20": 0.9547832679237122,
+                "ndcg@40": 0.9627457241783169
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "ndcg@10": 0.9290942178703699,
+                "ndcg@20": 0.9467035648480672,
+                "ndcg@40": 0.9563220304481116
+            }
+        }
+    }
+}

result/Retrieval/scores_jagovfaqs_22k.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.7455660589538348,
+    "details": {
+        "optimal_distance_metric": "euclidean_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.6042702544603685,
+                "accuracy@3": 0.7853173442527055,
+                "accuracy@5": 0.830944720678561,
+                "accuracy@10": 0.8821292775665399,
+                "ndcg@10": 0.7477862730518441,
+                "mrr@10": 0.7043207426287267
+            },
+            "dot_score": {
+                "accuracy@1": 0.4597835624451594,
+                "accuracy@3": 0.6607195086282539,
+                "accuracy@5": 0.7282831237203861,
+                "accuracy@10": 0.80549868382568,
+                "ndcg@10": 0.630976061323317,
+                "mrr@10": 0.5752777429583498
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.6092424685580579,
+                "accuracy@3": 0.7861947937993565,
+                "accuracy@5": 0.8283123720386077,
+                "accuracy@10": 0.8780345130155016,
+                "ndcg@10": 0.7480985513112418,
+                "mrr@10": 0.7060561428432148
+            }
+        },
+        "test_scores": {
+            "euclidean_distance": {
+                "accuracy@1": 0.6035087719298246,
+                "accuracy@3": 0.7795321637426901,
+                "accuracy@5": 0.8277777777777777,
+                "accuracy@10": 0.881578947368421,
+                "ndcg@10": 0.7455660589538348,
+                "mrr@10": 0.7017308317089019
+            }
+        }
+    }
+}

result/Retrieval/scores_jaqket.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.5012253145754781,
+    "details": {
+        "optimal_distance_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.3407035175879397,
+                "accuracy@3": 0.521608040201005,
+                "accuracy@5": 0.6040201005025125,
+                "accuracy@10": 0.6894472361809045,
+                "ndcg@10": 0.5074962109064866,
+                "mrr@10": 0.44994017707585504
+            },
+            "dot_score": {
+                "accuracy@1": 0.31055276381909547,
+                "accuracy@3": 0.507537688442211,
+                "accuracy@5": 0.5738693467336683,
+                "accuracy@10": 0.6804020100502512,
+                "ndcg@10": 0.48656131133927916,
+                "mrr@10": 0.42555116854111785
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.3055276381909548,
+                "accuracy@3": 0.4814070351758794,
+                "accuracy@5": 0.5597989949748744,
+                "accuracy@10": 0.6391959798994975,
+                "ndcg@10": 0.4655083260444005,
+                "mrr@10": 0.4106070032703195
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.3159478435305918,
+                "accuracy@3": 0.526579739217653,
+                "accuracy@5": 0.60481444332999,
+                "accuracy@10": 0.6920762286860582,
+                "ndcg@10": 0.5012253145754781,
+                "mrr@10": 0.4404156915190016
+            }
+        }
+    }
+}

result/Retrieval/scores_mrtydi.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.3545113073009125,
+    "details": {
+        "optimal_distance_metric": "euclidean_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.22306034482758622,
+                "accuracy@3": 0.37176724137931033,
+                "accuracy@5": 0.4536637931034483,
+                "accuracy@10": 0.5549568965517241,
+                "ndcg@10": 0.37815020333355365,
+                "mrr@10": 0.3228995621236997
+            },
+            "dot_score": {
+                "accuracy@1": 0.13793103448275862,
+                "accuracy@3": 0.2704741379310345,
+                "accuracy@5": 0.3394396551724138,
+                "accuracy@10": 0.4170258620689655,
+                "ndcg@10": 0.2698064952674162,
+                "mrr@10": 0.22368979200875752
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.22844827586206898,
+                "accuracy@3": 0.38362068965517243,
+                "accuracy@5": 0.4665948275862069,
+                "accuracy@10": 0.5668103448275862,
+                "ndcg@10": 0.38745306818571434,
+                "mrr@10": 0.33128378147235893
+            }
+        },
+        "test_scores": {
+            "euclidean_distance": {
+                "accuracy@1": 0.23194444444444445,
+                "accuracy@3": 0.3888888888888889,
+                "accuracy@5": 0.46805555555555556,
+                "accuracy@10": 0.5708333333333333,
+                "ndcg@10": 0.3545113073009125,
+                "mrr@10": 0.3320238095238095
+            }
+        }
+    }
+}

result/Retrieval/scores_nlp_journal_abs_intro.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.8689204088388403,
+    "details": {
+        "optimal_distance_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.85,
+                "accuracy@3": 0.93,
+                "accuracy@5": 0.93,
+                "accuracy@10": 0.95,
+                "ndcg@10": 0.9031188595062929,
+                "mrr@10": 0.8877777777777779
+            },
+            "dot_score": {
+                "accuracy@1": 0.75,
+                "accuracy@3": 0.87,
+                "accuracy@5": 0.88,
+                "accuracy@10": 0.91,
+                "ndcg@10": 0.8329701303885662,
+                "mrr@10": 0.8079563492063491
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.83,
+                "accuracy@3": 0.92,
+                "accuracy@5": 0.93,
+                "accuracy@10": 0.94,
+                "ndcg@10": 0.8903171995628786,
+                "mrr@10": 0.87375
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.7945544554455446,
+                "accuracy@3": 0.8836633663366337,
+                "accuracy@5": 0.9084158415841584,
+                "accuracy@10": 0.943069306930693,
+                "ndcg@10": 0.8689204088388403,
+                "mrr@10": 0.8452508643721514
+            }
+        }
+    }
+}

result/Retrieval/scores_nlp_journal_title_abs.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.9656989703684407,
+    "details": {
+        "optimal_distance_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.9,
+                "accuracy@3": 0.96,
+                "accuracy@5": 0.98,
+                "accuracy@10": 0.99,
+                "ndcg@10": 0.9477320812882918,
+                "mrr@10": 0.9339444444444445
+            },
+            "dot_score": {
+                "accuracy@1": 0.82,
+                "accuracy@3": 0.92,
+                "accuracy@5": 0.94,
+                "accuracy@10": 0.96,
+                "ndcg@10": 0.8940025955079818,
+                "mrr@10": 0.8724285714285713
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.89,
+                "accuracy@3": 0.97,
+                "accuracy@5": 0.98,
+                "accuracy@10": 0.99,
+                "ndcg@10": 0.9453171995628784,
+                "mrr@10": 0.9304166666666666
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.9306930693069307,
+                "accuracy@3": 0.9777227722772277,
+                "accuracy@5": 0.9876237623762376,
+                "accuracy@10": 0.995049504950495,
+                "ndcg@10": 0.9656989703684407,
+                "mrr@10": 0.955987741631306
+            }
+        }
+    }
+}

result/Retrieval/scores_nlp_journal_title_intro.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.7531306059721564,
+    "details": {
+        "optimal_distance_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.57,
+                "accuracy@3": 0.8,
+                "accuracy@5": 0.83,
+                "accuracy@10": 0.9,
+                "ndcg@10": 0.7448902792577736,
+                "mrr@10": 0.6942023809523811
+            },
+            "dot_score": {
+                "accuracy@1": 0.49,
+                "accuracy@3": 0.68,
+                "accuracy@5": 0.71,
+                "accuracy@10": 0.83,
+                "ndcg@10": 0.6537395005077568,
+                "mrr@10": 0.5984801587301588
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.58,
+                "accuracy@3": 0.75,
+                "accuracy@5": 0.85,
+                "accuracy@10": 0.9,
+                "ndcg@10": 0.7411266935263704,
+                "mrr@10": 0.6896904761904763
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.6237623762376238,
+                "accuracy@3": 0.7896039603960396,
+                "accuracy@5": 0.8242574257425742,
+                "accuracy@10": 0.8811881188118812,
+                "ndcg@10": 0.7531306059721564,
+                "mrr@10": 0.7120059327361306
+            }
+        }
+    }
+}

result/STS/scores_jsick.json

@@ -0,0 +1,31 @@
+{
+    "metric_name": "spearman",
+    "metric_value": 0.8231772134744029,
+    "details": {
+        "optimal_similarity_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "pearson": 0.8390312744889947,
+                "spearman": 0.8309726355825223
+            },
+            "manhatten_distance": {
+                "pearson": 0.8439757378089565,
+                "spearman": 0.8296746939532708
+            },
+            "euclidean_distance": {
+                "pearson": 0.8439757378089565,
+                "spearman": 0.8296746939532708
+            },
+            "dot_score": {
+                "pearson": 0.8235943624962084,
+                "spearman": 0.8066842966908715
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "pearson": 0.8323321086750828,
+                "spearman": 0.8231772134744029
+            }
+        }
+    }
+}

result/STS/scores_jsts.json

@@ -0,0 +1,31 @@
+{
+    "metric_name": "spearman",
+    "metric_value": 0.8342848039994751,
+    "details": {
+        "optimal_similarity_metric": "manhatten_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "pearson": 0.8402004412140045,
+                "spearman": 0.7947630577888891
+            },
+            "manhatten_distance": {
+                "pearson": 0.8359705278620446,
+                "spearman": 0.7954996671020325
+            },
+            "euclidean_distance": {
+                "pearson": 0.8359705278620446,
+                "spearman": 0.7954996671020325
+            },
+            "dot_score": {
+                "pearson": 0.8146522053769387,
+                "spearman": 0.7576805023715597
+            }
+        },
+        "test_scores": {
+            "manhatten_distance": {
+                "pearson": 0.8665411120423515,
+                "spearman": 0.8342848039994751
+            }
+        }
+    }
+}

result/summary.json

@@ -0,0 +1,62 @@
+{
+    "Classification": {
+        "amazon_counterfactual_classification": {
+            "macro_f1": 0.7665550732749669
+        },
+        "amazon_review_classification": {
+            "macro_f1": 0.5575876111411316
+        },
+        "massive_intent_classification": {
+            "macro_f1": 0.8141210121425055
+        },
+        "massive_scenario_classification": {
+            "macro_f1": 0.8848812917656395
+        }
+    },
+    "Reranking": {
+        "esci": {
+            "ndcg@10": 0.9290942178703699
+        }
+    },
+    "Retrieval": {
+        "jagovfaqs_22k": {
+            "ndcg@10": 0.7455660589538348
+        },
+        "jaqket": {
+            "ndcg@10": 0.5012253145754781
+        },
+        "mrtydi": {
+            "ndcg@10": 0.3545113073009125
+        },
+        "nlp_journal_abs_intro": {
+            "ndcg@10": 0.8689204088388403
+        },
+        "nlp_journal_title_abs": {
+            "ndcg@10": 0.9656989703684407
+        },
+        "nlp_journal_title_intro": {
+            "ndcg@10": 0.7531306059721564
+        }
+    },
+    "STS": {
+        "jsick": {
+            "spearman": 0.8231772134744029
+        },
+        "jsts": {
+            "spearman": 0.8342848039994751
+        }
+    },
+    "Clustering": {
+        "livedoor_news": {
+            "v_measure_score": 0.5427223607801758
+        },
+        "mewsc16": {
+            "v_measure_score": 0.5404099864321413
+        }
+    },
+    "PairClassification": {
+        "paws_x_ja": {
+            "binary_f1": 0.6237623762376238
+        }
+    }
+}