Spaces:
Runtime error
Runtime error
| conditions: | |
| # mDPR, tied encoders, pFT w/ MS MARCO, FT all | |
| - name: mdpr-tied-pft-msmarco-ft-all.ar | |
| eval_key: mrtydi-v1.1-arabic | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-arabic-${split} --index mrtydi-v1.1-arabic-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.9505 | |
| R@100: 1.0000 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.6913 | |
| R@100: 0.9165 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.6949 | |
| R@100: 0.9004 | |
| - name: mdpr-tied-pft-msmarco-ft-all.bn | |
| eval_key: mrtydi-v1.1-bengali | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-bengali-${split} --index mrtydi-v1.1-bengali-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.9620 | |
| R@100: 1.0000 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.5897 | |
| R@100: 0.8977 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.6228 | |
| R@100: 0.9550 | |
| - name: mdpr-tied-pft-msmarco-ft-all.en | |
| eval_key: mrtydi-v1.1-english | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-english-${split} --index mrtydi-v1.1-english-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.8278 | |
| R@100: 1.0000 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.5357 | |
| R@100: 0.8884 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.4916 | |
| R@100: 0.8414 | |
| - name: mdpr-tied-pft-msmarco-ft-all.fi | |
| eval_key: mrtydi-v1.1-finnish | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-finnish-${split} --index mrtydi-v1.1-finnish-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.9577 | |
| R@100: 0.9997 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.6626 | |
| R@100: 0.9171 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.5595 | |
| R@100: 0.8563 | |
| - name: mdpr-tied-pft-msmarco-ft-all.id | |
| eval_key: mrtydi-v1.1-indonesian | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-indonesian-${split} --index mrtydi-v1.1-indonesian-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.9469 | |
| R@100: 1.0000 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.6294 | |
| R@100: 0.9150 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.5783 | |
| R@100: 0.8609 | |
| - name: mdpr-tied-pft-msmarco-ft-all.ja | |
| eval_key: mrtydi-v1.1-japanese | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-japanese-${split} --index mrtydi-v1.1-japanese-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.8802 | |
| R@100: 0.9997 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.5505 | |
| R@100: 0.8696 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.5007 | |
| R@100: 0.8130 | |
| - name: mdpr-tied-pft-msmarco-ft-all.ko | |
| eval_key: mrtydi-v1.1-korean | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-korean-${split} --index mrtydi-v1.1-korean-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.9195 | |
| R@100: 1.0000 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.5645 | |
| R@100: 0.8663 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.4861 | |
| R@100: 0.7854 | |
| - name: mdpr-tied-pft-msmarco-ft-all.ru | |
| eval_key: mrtydi-v1.1-russian | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-russian-${split} --index mrtydi-v1.1-russian-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.8473 | |
| R@100: 0.9994 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.5104 | |
| R@100: 0.8720 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.5161 | |
| R@100: 0.8432 | |
| - name: mdpr-tied-pft-msmarco-ft-all.sw | |
| eval_key: mrtydi-v1.1-swahili | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-swahili-${split} --index mrtydi-v1.1-swahili-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.9515 | |
| R@100: 1.0000 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.6404 | |
| R@100: 0.9018 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.6438 | |
| R@100: 0.8756 | |
| - name: mdpr-tied-pft-msmarco-ft-all.te | |
| eval_key: mrtydi-v1.1-telugu | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-telugu-${split} --index mrtydi-v1.1-telugu-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.9679 | |
| R@100: 1.0000 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.7962 | |
| R@100: 0.9593 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.8908 | |
| R@100: 0.9659 | |
| - name: mdpr-tied-pft-msmarco-ft-all.th | |
| eval_key: mrtydi-v1.1-thai | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco-ft-all --topics mrtydi-v1.1-thai-${split} --index mrtydi-v1.1-thai-mdpr-tied-pft-msmarco-ft-all --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.9504 | |
| R@100: 1.0000 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.6670 | |
| R@100: 0.9114 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.6175 | |
| R@100: 0.8826 | |
| # mDPR, tied encoders, pFT w/ MS MARCO | |
| - name: mdpr-tied-pft-msmarco.ar | |
| eval_key: mrtydi-v1.1-arabic | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-arabic-${split} --index mrtydi-v1.1-arabic-mdpr-tied-pft-msmarco --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.3957 | |
| R@100: 0.7818 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.3978 | |
| R@100: 0.7778 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.4414 | |
| R@100: 0.7971 | |
| - name: mdpr-tied-pft-msmarco.bn | |
| eval_key: mrtydi-v1.1-bengali | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-bengali-${split} --index mrtydi-v1.1-bengali-mdpr-tied-pft-msmarco --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2920 | |
| R@100: 0.7323 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2993 | |
| R@100: 0.7318 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.3969 | |
| R@100: 0.7838 | |
| - name: mdpr-tied-pft-msmarco.en | |
| eval_key: mrtydi-v1.1-english | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-english-${split} --index mrtydi-v1.1-english-mdpr-tied-pft-msmarco --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.3374 | |
| R@100: 0.8111 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.3451 | |
| R@100: 0.7995 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.3270 | |
| R@100: 0.7536 | |
| - name: mdpr-tied-pft-msmarco.fi | |
| eval_key: mrtydi-v1.1-finnish | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-finnish-${split} --index mrtydi-v1.1-finnish-mdpr-tied-pft-msmarco --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.3668 | |
| R@100: 0.7337 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.3636 | |
| R@100: 0.7371 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2750 | |
| R@100: 0.6471 | |
| - name: mdpr-tied-pft-msmarco.id | |
| eval_key: mrtydi-v1.1-indonesian | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-indonesian-${split} --index mrtydi-v1.1-indonesian-mdpr-tied-pft-msmarco --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2794 | |
| R@100: 0.7044 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2853 | |
| R@100: 0.7198 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.3520 | |
| R@100: 0.7356 | |
| - name: mdpr-tied-pft-msmarco.ja | |
| eval_key: mrtydi-v1.1-japanese | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-japanese-${split} --index mrtydi-v1.1-japanese-mdpr-tied-pft-msmarco --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.3089 | |
| R@100: 0.7603 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.3108 | |
| R@100: 0.7597 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.3107 | |
| R@100: 0.7317 | |
| - name: mdpr-tied-pft-msmarco.ko | |
| eval_key: mrtydi-v1.1-korean | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-korean-${split} --index mrtydi-v1.1-korean-mdpr-tied-pft-msmarco --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.3003 | |
| R@100: 0.6907 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.3017 | |
| R@100: 0.7046 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2820 | |
| R@100: 0.6172 | |
| - name: mdpr-tied-pft-msmarco.ru | |
| eval_key: mrtydi-v1.1-russian | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-russian-${split} --index mrtydi-v1.1-russian-mdpr-tied-pft-msmarco --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2856 | |
| R@100: 0.7305 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2943 | |
| R@100: 0.7404 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.3561 | |
| R@100: 0.7432 | |
| - name: mdpr-tied-pft-msmarco.sw | |
| eval_key: mrtydi-v1.1-swahili | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-swahili-${split} --index mrtydi-v1.1-swahili-mdpr-tied-pft-msmarco --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2491 | |
| R@100: 0.5195 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2447 | |
| R@100: 0.5266 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.3418 | |
| R@100: 0.6343 | |
| - name: mdpr-tied-pft-msmarco.te | |
| eval_key: mrtydi-v1.1-telugu | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-telugu-${split} --index mrtydi-v1.1-telugu-mdpr-tied-pft-msmarco --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.3059 | |
| R@100: 0.7510 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2995 | |
| R@100: 0.7355 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.3102 | |
| R@100: 0.7817 | |
| - name: mdpr-tied-pft-msmarco.th | |
| eval_key: mrtydi-v1.1-thai | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-msmarco --topics mrtydi-v1.1-thai-${split} --index mrtydi-v1.1-thai-mdpr-tied-pft-msmarco --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2334 | |
| R@100: 0.5851 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2407 | |
| R@100: 0.5795 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2693 | |
| R@100: 0.5945 | |
| # mDPR, tied encoders, pFT w/ NQ | |
| - name: mdpr-tied-pft-nq.ar | |
| eval_key: mrtydi-v1.1-arabic | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-arabic-${split} --index mrtydi-v1.1-arabic-mdpr-tied-pft-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2087 | |
| R@100: 0.5854 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2132 | |
| R@100: 0.5868 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2214 | |
| R@100: 0.6001 | |
| - name: mdpr-tied-pft-nq.bn | |
| eval_key: mrtydi-v1.1-bengali | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-bengali-${split} --index mrtydi-v1.1-bengali-mdpr-tied-pft-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2371 | |
| R@100: 0.6281 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2414 | |
| R@100: 0.6409 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2535 | |
| R@100: 0.7072 | |
| - name: mdpr-tied-pft-nq.en | |
| eval_key: mrtydi-v1.1-english | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-english-${split} --index mrtydi-v1.1-english-mdpr-tied-pft-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2441 | |
| R@100: 0.7217 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2359 | |
| R@100: 0.7187 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2433 | |
| R@100: 0.6893 | |
| - name: mdpr-tied-pft-nq.fi | |
| eval_key: mrtydi-v1.1-finnish | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-finnish-${split} --index mrtydi-v1.1-finnish-mdpr-tied-pft-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2996 | |
| R@100: 0.6787 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.3252 | |
| R@100: 0.7037 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2444 | |
| R@100: 0.6401 | |
| - name: mdpr-tied-pft-nq.id | |
| eval_key: mrtydi-v1.1-indonesian | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-indonesian-${split} --index mrtydi-v1.1-indonesian-mdpr-tied-pft-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2706 | |
| R@100: 0.7322 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2719 | |
| R@100: 0.7394 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2815 | |
| R@100: 0.6914 | |
| - name: mdpr-tied-pft-nq.ja | |
| eval_key: mrtydi-v1.1-japanese | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-japanese-${split} --index mrtydi-v1.1-japanese-mdpr-tied-pft-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2165 | |
| R@100: 0.6043 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2299 | |
| R@100: 0.6239 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2058 | |
| R@100: 0.5734 | |
| - name: mdpr-tied-pft-nq.ko | |
| eval_key: mrtydi-v1.1-korean | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-korean-${split} --index mrtydi-v1.1-korean-mdpr-tied-pft-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2527 | |
| R@100: 0.6556 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2680 | |
| R@100: 0.6271 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2234 | |
| R@100: 0.5499 | |
| - name: mdpr-tied-pft-nq.ru | |
| eval_key: mrtydi-v1.1-russian | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-russian-${split} --index mrtydi-v1.1-russian-mdpr-tied-pft-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2160 | |
| R@100: 0.6262 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2263 | |
| R@100: 0.6444 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2501 | |
| R@100: 0.6181 | |
| - name: mdpr-tied-pft-nq.sw | |
| eval_key: mrtydi-v1.1-swahili | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-swahili-${split} --index mrtydi-v1.1-swahili-mdpr-tied-pft-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2383 | |
| R@100: 0.5707 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2543 | |
| R@100: 0.6138 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2621 | |
| R@100: 0.5965 | |
| - name: mdpr-tied-pft-nq.te | |
| eval_key: mrtydi-v1.1-telugu | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-telugu-${split} --index mrtydi-v1.1-telugu-mdpr-tied-pft-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.1483 | |
| R@100: 0.4162 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.1494 | |
| R@100: 0.3967 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.0970 | |
| R@100: 0.2454 | |
| - name: mdpr-tied-pft-nq.th | |
| eval_key: mrtydi-v1.1-thai | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder-class auto --encoder castorini/mdpr-tied-pft-nq --topics mrtydi-v1.1-thai-${split} --index mrtydi-v1.1-thai-mdpr-tied-pft-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.1426 | |
| R@100: 0.4717 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.1618 | |
| R@100: 0.4637 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.1575 | |
| R@100: 0.4550 | |
| # mDPR, split encoders, pFT w/ NQ | |
| - name: mdpr-split-pft-nq.ar | |
| eval_key: mrtydi-v1.1-arabic | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-arabic-${split} --index mrtydi-v1.1-arabic-mdpr-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2510 | |
| R@100: 0.6384 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2449 | |
| R@100: 0.6334 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2907 | |
| R@100: 0.6502 | |
| - name: mdpr-split-pft-nq.bn | |
| eval_key: mrtydi-v1.1-bengali | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-bengali-${split} --index mrtydi-v1.1-bengali-mdpr-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2293 | |
| R@100: 0.6454 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2367 | |
| R@100: 0.6511 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2911 | |
| R@100: 0.7793 | |
| - name: mdpr-split-pft-nq.en | |
| eval_key: mrtydi-v1.1-english | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-english-${split} --index mrtydi-v1.1-english-mdpr-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2862 | |
| R@100: 0.7372 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2821 | |
| R@100: 0.7437 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2907 | |
| R@100: 0.6779 | |
| - name: mdpr-split-pft-nq.fi | |
| eval_key: mrtydi-v1.1-finnish | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-finnish-${split} --index mrtydi-v1.1-finnish-mdpr-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2473 | |
| R@100: 0.6289 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2466 | |
| R@100: 0.6283 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2050 | |
| R@100: 0.5680 | |
| - name: mdpr-split-pft-nq.id | |
| eval_key: mrtydi-v1.1-indonesian | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-indonesian-${split} --index mrtydi-v1.1-indonesian-mdpr-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2351 | |
| R@100: 0.6952 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2475 | |
| R@100: 0.7181 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2705 | |
| R@100: 0.6848 | |
| - name: mdpr-split-pft-nq.ja | |
| eval_key: mrtydi-v1.1-japanese | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-japanese-${split} --index mrtydi-v1.1-japanese-mdpr-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.1967 | |
| R@100: 0.5983 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2055 | |
| R@100: 0.6142 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2119 | |
| R@100: 0.5840 | |
| - name: mdpr-split-pft-nq.ko | |
| eval_key: mrtydi-v1.1-korean | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-korean-${split} --index mrtydi-v1.1-korean-mdpr-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2383 | |
| R@100: 0.6180 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2343 | |
| R@100: 0.6238 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2345 | |
| R@100: 0.5325 | |
| - name: mdpr-split-pft-nq.ru | |
| eval_key: mrtydi-v1.1-russian | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-russian-${split} --index mrtydi-v1.1-russian-mdpr-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2317 | |
| R@100: 0.6534 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2490 | |
| R@100: 0.6553 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2820 | |
| R@100: 0.6474 | |
| - name: mdpr-split-pft-nq.sw | |
| eval_key: mrtydi-v1.1-swahili | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-swahili-${split} --index mrtydi-v1.1-swahili-mdpr-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.1457 | |
| R@100: 0.4481 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.1547 | |
| R@100: 0.4724 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.1883 | |
| R@100: 0.5281 | |
| - name: mdpr-split-pft-nq.te | |
| eval_key: mrtydi-v1.1-telugu | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-telugu-${split} --index mrtydi-v1.1-telugu-mdpr-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.1489 | |
| R@100: 0.4905 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.1503 | |
| R@100: 0.4934 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.1099 | |
| R@100: 0.3661 | |
| - name: mdpr-split-pft-nq.th | |
| eval_key: mrtydi-v1.1-thai | |
| command: python -m pyserini.search.faiss --threads 16 --batch-size 512 --encoder castorini/mdpr-question-nq --topics mrtydi-v1.1-thai-${split} --index mrtydi-v1.1-thai-mdpr-nq --output $output --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.1603 | |
| R@100: 0.4983 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.1584 | |
| R@100: 0.5083 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.1709 | |
| R@100: 0.5146 | |
| # BM25 | |
| - name: bm25.ar | |
| eval_key: mrtydi-v1.1-arabic | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language ar --topics mrtydi-v1.1-arabic-${split} --index mrtydi-v1.1-arabic --output $output --bm25 --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.3356 | |
| R@100: 0.7944 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.3462 | |
| R@100: 0.7872 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.3682 | |
| R@100: 0.7928 | |
| - name: bm25.bn | |
| eval_key: mrtydi-v1.1-bengali | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language bn --topics mrtydi-v1.1-bengali-${split} --index mrtydi-v1.1-bengali --output $output --bm25 --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.3566 | |
| - R@100: 0.8336 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.3385 | |
| - R@100: 0.8432 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.4182 | |
| - R@100: 0.8694 | |
| - name: bm25.en | |
| eval_key: mrtydi-v1.1-english | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language en --topics mrtydi-v1.1-english-${split} --index mrtydi-v1.1-english --output $output --bm25 --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.1592 | |
| - R@100: 0.5785 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.1685 | |
| - R@100: 0.6196 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.1404 | |
| - R@100: 0.5365 | |
| - name: bm25.fi | |
| eval_key: mrtydi-v1.1-finnish | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language fi --topics mrtydi-v1.1-finnish-${split} --index mrtydi-v1.1-finnish --output $output --bm25 --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.4101 | |
| - R@100: 0.8198 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.4136 | |
| - R@100: 0.8285 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2836 | |
| - R@100: 0.7196 | |
| - name: bm25.id | |
| eval_key: mrtydi-v1.1-indonesian | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language id --topics mrtydi-v1.1-indonesian-${split} --index mrtydi-v1.1-indonesian --output $output --bm25 --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2972 | |
| - R@100: 0.7948 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2937 | |
| - R@100: 0.7827 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.3762 | |
| - R@100: 0.8426 | |
| - name: bm25.ja | |
| eval_key: mrtydi-v1.1-japanese | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language ja --topics mrtydi-v1.1-japanese-${split} --index mrtydi-v1.1-japanese --output $output --bm25 --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2262 | |
| - R@100: 0.7290 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2250 | |
| - R@100: 0.7252 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2125 | |
| - R@100: 0.6431 | |
| - name: bm25.ko | |
| eval_key: mrtydi-v1.1-korean | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language ko --topics mrtydi-v1.1-korean-${split} --index mrtydi-v1.1-korean --output $output --bm25 --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2596 | |
| - R@100: 0.6178 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2888 | |
| - R@100: 0.6733 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.2848 | |
| - R@100: 0.6188 | |
| - name: bm25.ru | |
| eval_key: mrtydi-v1.1-russian | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language ru --topics mrtydi-v1.1-russian-${split} --index mrtydi-v1.1-russian --output $output --bm25 --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2229 | |
| - R@100: 0.5779 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2202 | |
| - R@100: 0.5760 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.3163 | |
| - R@100: 0.6541 | |
| - name: bm25.sw | |
| eval_key: mrtydi-v1.1-swahili | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language sw --topics mrtydi-v1.1-swahili-${split} --index mrtydi-v1.1-swahili --output $output --bm25 --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.2610 | |
| - R@100: 0.5903 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.2693 | |
| - R@100: 0.5789 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.3893 | |
| - R@100: 0.7642 | |
| - name: bm25.te | |
| eval_key: mrtydi-v1.1-telugu | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language te --topics mrtydi-v1.1-telugu-${split} --index mrtydi-v1.1-telugu --output $output --bm25 --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.4204 | |
| - R@100: 0.8229 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.4269 | |
| - R@100: 0.8362 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.5283 | |
| - R@100: 0.8971 | |
| - name: bm25.th | |
| eval_key: mrtydi-v1.1-thai | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --language th --topics mrtydi-v1.1-thai-${split} --index mrtydi-v1.1-thai --output $output --bm25 --hits 100 | |
| splits: | |
| - split: train | |
| scores: | |
| - MRR@100: 0.3543 | |
| - R@100: 0.8349 | |
| - split: dev | |
| scores: | |
| - MRR@100: 0.3586 | |
| - R@100: 0.8536 | |
| - split: test | |
| scores: | |
| - MRR@100: 0.4012 | |
| - R@100: 0.8529 | |