WariHima/hourai2-10m-64dim-embedding

蓬莱2 10m 64dim emmbedding

hourai2 10 emmneddingの構成をベースに、
hiden_sizeを128->64に変更し、レイヤー数を14->28に増やしたものです。
同じ条件で学習し、128dim モデル(hourai2 10membedding)とほぼ同じスコアを出しました。
速度は128dimモデルより２倍ほど遅いですが、その分、小メモリです。
ただ、速度、メモリ使用量ともに空気のようなサイズなので、正確に測定しづらかったです。
100個以上メモリに乗せるような用途(専用サーバーを作るとか)でなく普通に使う場合は、
128dimモデルがおすすめです。

architecture:

• qwen3next
• hidden size 64

tarined by:

• rtx3060 * 3.15h training code in ./training_codes

tokenizer from ku-nlp/gpt2-small-japanese-char

dataset:

• hotchpotch/sentence_transformer_japanese use llmjp-kaken 30k rows 1 epoch

evals:

{
  "dataset_revision": "5bac629e25799df4c9c80a6a5db983d6cba9e77d",
  "task_name": "JSTS",
  "mteb_version": "2.18.5",
  "scores": {
    "validation": [
      {
        "pearson": 0.40546,
        "spearman": 0.397042,
        "cosine_pearson": 0.40546,
        "cosine_spearman": 0.397042,
        "manhattan_pearson": 0.422271,
        "manhattan_spearman": 0.397251,
        "euclidean_pearson": 0.423019,
        "euclidean_spearman": 0.397011,
        "main_score": 0.397042,
        "hf_subset": "default",
        "languages": [
          "jpn-Jpan"
        ],
        "mteb_version": "2.18.5"
      }
    ]
  },
  "evaluation_time": 45.2427179813385,
  "kg_co2_emissions": null,
  "date": null,
  "evaluation_phases": null
}

{
  "dataset_revision": "5713cd1b76a22b79c3595827c717c72b6cab8852",
  "task_name": "JaQuADRetrieval",
  "mteb_version": "2.18.5",
  "scores": {
    "validation": [
      {
        "ndcg_at_1": 0.01514,
        "ndcg_at_3": 0.01598,
        "ndcg_at_5": 0.01997,
        "ndcg_at_10": 0.0272,
        "ndcg_at_20": 0.03734,
        "ndcg_at_100": 0.06783,
        "ndcg_at_1000": 0.14079,
        "map_at_1": 0.00757,
        "map_at_3": 0.0118,
        "map_at_5": 0.01362,
        "map_at_10": 0.01619,
        "map_at_20": 0.01861,
        "map_at_100": 0.02229,
        "map_at_1000": 0.02464,
        "recall_at_1": 0.00757,
        "recall_at_3": 0.01709,
        "recall_at_5": 0.02515,
        "recall_at_10": 0.04346,
        "recall_at_20": 0.07666,
        "recall_at_100": 0.21606,
        "recall_at_1000": 0.71606,
        "accuracy": 0.00757,
        "precision_at_1": 0.01514,
        "precision_at_3": 0.01139,
        "precision_at_5": 0.01006,
        "precision_at_10": 0.00869,
        "precision_at_20": 0.00767,
        "precision_at_100": 0.00432,
        "precision_at_1000": 0.00143,
        "mrr_at_1": 0.015137,
        "mrr_at_3": 0.022949,
        "mrr_at_5": 0.026392,
        "mrr_at_10": 0.03091,
        "mrr_at_20": 0.03487,
        "mrr_at_100": 0.039832,
        "mrr_at_1000": 0.041932,
        "nauc_ndcg_at_1_max": 0.093331,
        "nauc_ndcg_at_1_std": 0.188348,
        "nauc_ndcg_at_1_diff1": 0.221412,
        "nauc_ndcg_at_3_max": 0.016638,
        "nauc_ndcg_at_3_std": 0.214451,
        "nauc_ndcg_at_3_diff1": 0.210734,
        "nauc_ndcg_at_5_max": -0.016221,
        "nauc_ndcg_at_5_std": 0.225589,
        "nauc_ndcg_at_5_diff1": 0.158523,
        "nauc_ndcg_at_10_max": -0.034748,
        "nauc_ndcg_at_10_std": 0.239215,
        "nauc_ndcg_at_10_diff1": 0.154824,
        "nauc_ndcg_at_20_max": -0.039591,
        "nauc_ndcg_at_20_std": 0.253405,
        "nauc_ndcg_at_20_diff1": 0.141955,
        "nauc_ndcg_at_100_max": -0.070595,
        "nauc_ndcg_at_100_std": 0.279888,
        "nauc_ndcg_at_100_diff1": 0.089933,
        "nauc_ndcg_at_1000_max": -0.057872,
        "nauc_ndcg_at_1000_std": 0.266175,
        "nauc_ndcg_at_1000_diff1": 0.089756,
        "nauc_map_at_1_max": 0.093331,
        "nauc_map_at_1_std": 0.188348,
        "nauc_map_at_1_diff1": 0.221412,
        "nauc_map_at_3_max": 0.026441,
        "nauc_map_at_3_std": 0.21993,
        "nauc_map_at_3_diff1": 0.216307,
        "nauc_map_at_5_max": 0.002704,
        "nauc_map_at_5_std": 0.228759,
        "nauc_map_at_5_diff1": 0.184053,
        "nauc_map_at_10_max": -0.014023,
        "nauc_map_at_10_std": 0.242162,
        "nauc_map_at_10_diff1": 0.183676,
        "nauc_map_at_20_max": -0.019069,
        "nauc_map_at_20_std": 0.250555,
        "nauc_map_at_20_diff1": 0.178857,
        "nauc_map_at_100_max": -0.032624,
        "nauc_map_at_100_std": 0.261637,
        "nauc_map_at_100_diff1": 0.156789,
        "nauc_map_at_1000_max": -0.033925,
        "nauc_map_at_1000_std": 0.263274,
        "nauc_map_at_1000_diff1": 0.156092,
        "nauc_recall_at_1_max": 0.093331,
        "nauc_recall_at_1_std": 0.188348,
        "nauc_recall_at_1_diff1": 0.221412,
        "nauc_recall_at_3_max": 0.001047,
        "nauc_recall_at_3_std": 0.208681,
        "nauc_recall_at_3_diff1": 0.201183,
        "nauc_recall_at_5_max": -0.046939,
        "nauc_recall_at_5_std": 0.229501,
        "nauc_recall_at_5_diff1": 0.121437,
        "nauc_recall_at_10_max": -0.063101,
        "nauc_recall_at_10_std": 0.247356,
        "nauc_recall_at_10_diff1": 0.130637,
        "nauc_recall_at_20_max": -0.062052,
        "nauc_recall_at_20_std": 0.26945,
        "nauc_recall_at_20_diff1": 0.117451,
        "nauc_recall_at_100_max": -0.098454,
        "nauc_recall_at_100_std": 0.29955,
        "nauc_recall_at_100_diff1": 0.043165,
        "nauc_recall_at_1000_max": -0.049005,
        "nauc_recall_at_1000_std": 0.219725,
        "nauc_recall_at_1000_diff1": 0.025471,
        "nauc_precision_at_1_max": 0.093331,
        "nauc_precision_at_1_std": 0.188348,
        "nauc_precision_at_1_diff1": 0.221412,
        "nauc_precision_at_3_max": 0.001047,
        "nauc_precision_at_3_std": 0.208681,
        "nauc_precision_at_3_diff1": 0.201183,
        "nauc_precision_at_5_max": -0.046939,
        "nauc_precision_at_5_std": 0.229501,
        "nauc_precision_at_5_diff1": 0.121437,
        "nauc_precision_at_10_max": -0.063101,
        "nauc_precision_at_10_std": 0.247356,
        "nauc_precision_at_10_diff1": 0.130637,
        "nauc_precision_at_20_max": -0.062052,
        "nauc_precision_at_20_std": 0.26945,
        "nauc_precision_at_20_diff1": 0.117451,
        "nauc_precision_at_100_max": -0.098454,
        "nauc_precision_at_100_std": 0.29955,
        "nauc_precision_at_100_diff1": 0.043165,
        "nauc_precision_at_1000_max": -0.049005,
        "nauc_precision_at_1000_std": 0.219725,
        "nauc_precision_at_1000_diff1": 0.025471,
        "nauc_mrr_at_1_max": 0.093331,
        "nauc_mrr_at_1_std": 0.188348,
        "nauc_mrr_at_1_diff1": 0.221412,
        "nauc_mrr_at_3_max": 0.02369,
        "nauc_mrr_at_3_std": 0.212986,
        "nauc_mrr_at_3_diff1": 0.214054,
        "nauc_mrr_at_5_max": 0.002104,
        "nauc_mrr_at_5_std": 0.217231,
        "nauc_mrr_at_5_diff1": 0.175259,
        "nauc_mrr_at_10_max": -0.008243,
        "nauc_mrr_at_10_std": 0.222218,
        "nauc_mrr_at_10_diff1": 0.165171,
        "nauc_mrr_at_20_max": -0.01045,
        "nauc_mrr_at_20_std": 0.225566,
        "nauc_mrr_at_20_diff1": 0.15538,
        "nauc_mrr_at_100_max": -0.018553,
        "nauc_mrr_at_100_std": 0.228237,
        "nauc_mrr_at_100_diff1": 0.143405,
        "nauc_mrr_at_1000_max": -0.018345,
        "nauc_mrr_at_1000_std": 0.228414,
        "nauc_mrr_at_1000_diff1": 0.144785,
        "hit_rate_at_1": 0.01514,
        "hit_rate_at_3": 0.0332,
        "hit_rate_at_5": 0.04883,
        "hit_rate_at_10": 0.08301,
        "hit_rate_at_20": 0.14258,
        "hit_rate_at_100": 0.35791,
        "hit_rate_at_1000": 0.89941,
        "main_score": 0.0272,
        "hf_subset": "default",
        "languages": [
          "jpn-Jpan"
        ],
        "mteb_version": "2.18.5"
      }
    ]
  },
  "evaluation_time": 82.70160269737244,
  "kg_co2_emissions": null,
  "date": null,
  "evaluation_phases": null
}