Spaces:
Runtime error
Runtime error
| conditions: | |
| - name: bm25-doc-default | |
| display: BM25 doc (k1=0.9, b=0.4) | |
| display-html: BM25 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (1a) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-slim --topics $topics --output $output --bm25 | |
| topics: | |
| - topic_key: msmarco-v2-doc-dev | |
| eval_key: msmarco-v2-doc-dev | |
| scores: | |
| - MRR@100: 0.1572 | |
| R@1K: 0.8054 | |
| - topic_key: msmarco-v2-doc-dev2 | |
| eval_key: msmarco-v2-doc-dev2 | |
| scores: | |
| - MRR@100: 0.1659 | |
| R@1K: 0.8029 | |
| - topic_key: dl21 | |
| eval_key: dl21-doc | |
| scores: | |
| - MAP@100: 0.2126 | |
| nDCG@10: 0.5116 | |
| MRR@100: 0.8367 | |
| R@100: 0.3195 | |
| R@1K: 0.6739 | |
| - name: bm25-doc-segmented-default | |
| display: BM25 doc segmented (k1=0.9, b=0.4) | |
| display-html: BM25 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (1b) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-slim --topics $topics --output $output --bm25 --hits 10000 --max-passage-hits 1000 --max-passage | |
| topics: | |
| - topic_key: msmarco-v2-doc-dev | |
| eval_key: msmarco-v2-doc-dev | |
| scores: | |
| - MRR@100: 0.1896 | |
| R@1K: 0.8542 | |
| - topic_key: msmarco-v2-doc-dev2 | |
| eval_key: msmarco-v2-doc-dev2 | |
| scores: | |
| - MRR@100: 0.1930 | |
| R@1K: 0.8549 | |
| - topic_key: dl21 | |
| eval_key: dl21-doc | |
| scores: | |
| - MAP@100: 0.2436 | |
| nDCG@10: 0.5776 | |
| MRR@100: 0.8937 | |
| R@100: 0.3478 | |
| R@1K: 0.6930 | |
| - name: bm25-rm3-doc-default | |
| display: BM25+RM3 doc (k1=0.9, b=0.4) | |
| display-html: BM25+RM3 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (1c) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-full --topics $topics --output $output --bm25 --rm3 | |
| topics: | |
| - topic_key: msmarco-v2-doc-dev | |
| eval_key: msmarco-v2-doc-dev | |
| scores: | |
| - MRR@100: 0.0974 | |
| R@1K: 0.7699 | |
| - topic_key: msmarco-v2-doc-dev2 | |
| eval_key: msmarco-v2-doc-dev2 | |
| scores: | |
| - MRR@100: 0.1033 | |
| R@1K: 0.7736 | |
| - topic_key: dl21 | |
| eval_key: dl21-doc | |
| scores: | |
| - MAP@100: 0.2452 | |
| nDCG@10: 0.5304 | |
| MRR@100: 0.7914 | |
| R@100: 0.3376 | |
| R@1K: 0.7341 | |
| - name: bm25-rm3-doc-segmented-default | |
| display: BM25+RM3 doc segmented (k1=0.9, b=0.4) | |
| display-html: BM25+RM3 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (1d) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-full --topics $topics --output $output --bm25 --rm3 --hits 10000 --max-passage-hits 1000 --max-passage | |
| topics: | |
| - topic_key: msmarco-v2-doc-dev | |
| eval_key: msmarco-v2-doc-dev | |
| scores: | |
| - MRR@100: 0.1660 | |
| R@1K: 0.8608 | |
| - topic_key: msmarco-v2-doc-dev2 | |
| eval_key: msmarco-v2-doc-dev2 | |
| scores: | |
| - MRR@100: 0.1702 | |
| R@1K: 0.8639 | |
| - topic_key: dl21 | |
| eval_key: dl21-doc | |
| scores: | |
| - MAP@100: 0.2936 | |
| nDCG@10: 0.6189 | |
| MRR@100: 0.9076 | |
| R@100: 0.3890 | |
| R@1K: 0.7678 | |
| - name: bm25-d2q-t5-doc-default | |
| display: BM25 w/ doc2query-T5 doc (k1=0.9, b=0.4) | |
| display-html: BM25 w/ doc2query-T5 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (2a) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-d2q-t5 --topics $topics --output $output --bm25 | |
| topics: | |
| - topic_key: msmarco-v2-doc-dev | |
| eval_key: msmarco-v2-doc-dev | |
| scores: | |
| - MRR@100: 0.2011 | |
| R@1K: 0.8614 | |
| - topic_key: msmarco-v2-doc-dev2 | |
| eval_key: msmarco-v2-doc-dev2 | |
| scores: | |
| - MRR@100: 0.2012 | |
| R@1K: 0.8568 | |
| - topic_key: dl21 | |
| eval_key: dl21-doc | |
| scores: | |
| - MAP@100: 0.2387 | |
| nDCG@10: 0.5792 | |
| MRR@100: 0.8866 | |
| R@100: 0.3443 | |
| R@1K: 0.7066 | |
| - name: bm25-d2q-t5-doc-segmented-default | |
| display: BM25 w/ doc2query-T5 doc segmented (k1=0.9, b=0.4) | |
| display-html: BM25 w/ doc2query-T5 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (2b) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-d2q-t5 --topics $topics --output $output --bm25 --hits 10000 --max-passage-hits 1000 --max-passage | |
| topics: | |
| - topic_key: msmarco-v2-doc-dev | |
| eval_key: msmarco-v2-doc-dev | |
| scores: | |
| - MRR@100: 0.2226 | |
| R@1K: 0.8982 | |
| - topic_key: msmarco-v2-doc-dev2 | |
| eval_key: msmarco-v2-doc-dev2 | |
| scores: | |
| - MRR@100: 0.2234 | |
| R@1K: 0.8952 | |
| - topic_key: dl21 | |
| eval_key: dl21-doc | |
| scores: | |
| - MAP@100: 0.2683 | |
| nDCG@10: 0.6289 | |
| MRR@100: 0.9454 | |
| R@100: 0.3656 | |
| R@1K: 0.7202 | |
| - name: bm25-rm3-d2q-t5-doc-default | |
| display: BM25+RM3 w/ doc2query-T5 doc (k1=0.9, b=0.4) | |
| display-html: BM25+RM3 w/ doc2query-T5 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (2c) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 | |
| topics: | |
| - topic_key: msmarco-v2-doc-dev | |
| eval_key: msmarco-v2-doc-dev | |
| scores: | |
| - MRR@100: 0.1141 | |
| R@1K: 0.8191 | |
| - topic_key: msmarco-v2-doc-dev2 | |
| eval_key: msmarco-v2-doc-dev2 | |
| scores: | |
| - MRR@100: 0.1170 | |
| R@1K: 0.8247 | |
| - topic_key: dl21 | |
| eval_key: dl21-doc | |
| scores: | |
| - MAP@100: 0.2611 | |
| nDCG@10: 0.5375 | |
| MRR@100: 0.8255 | |
| R@100: 0.3580 | |
| R@1K: 0.7574 | |
| - name: bm25-rm3-d2q-t5-doc-segmented-default | |
| display: BM25+RM3 w/ doc2query-T5 doc segmented (k1=0.9, b=0.4) | |
| display-html: BM25+RM3 w/ doc2query-T5 doc segmented (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4) | |
| display-row: (2d) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-d2q-t5-docvectors --topics $topics --output $output --bm25 --rm3 --hits 10000 --max-passage-hits 1000 --max-passage | |
| topics: | |
| - topic_key: msmarco-v2-doc-dev | |
| eval_key: msmarco-v2-doc-dev | |
| scores: | |
| - MRR@100: 0.1975 | |
| R@1K: 0.9002 | |
| - topic_key: msmarco-v2-doc-dev2 | |
| eval_key: msmarco-v2-doc-dev2 | |
| scores: | |
| - MRR@100: 0.1978 | |
| R@1K: 0.8972 | |
| - topic_key: dl21 | |
| eval_key: dl21-doc | |
| scores: | |
| - MAP@100: 0.3191 | |
| nDCG@10: 0.6559 | |
| MRR@100: 0.8989 | |
| R@100: 0.4131 | |
| R@1K: 0.7948 | |
| - name: unicoil-noexp | |
| display: "uniCOIL (noexp): pre-encoded" | |
| display-html: "uniCOIL (noexp): pre-encoded queries" | |
| display-row: (3a) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-noexp-0shot --topics $topics --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage | |
| topics: | |
| - topic_key: msmarco-v2-doc-dev-unicoil-noexp | |
| eval_key: msmarco-v2-doc-dev | |
| scores: | |
| - MRR@100: 0.2231 | |
| R@1K: 0.8987 | |
| - topic_key: msmarco-v2-doc-dev2-unicoil-noexp | |
| eval_key: msmarco-v2-doc-dev2 | |
| scores: | |
| - MRR@100: 0.2314 | |
| R@1K: 0.8995 | |
| - topic_key: dl21-unicoil-noexp | |
| eval_key: dl21-doc | |
| scores: | |
| - MAP@100: 0.2587 | |
| nDCG@10: 0.6495 | |
| MRR@100: 0.9282 | |
| R@100: 0.3563 | |
| R@1K: 0.6787 | |
| - name: unicoil-noexp-otf | |
| display: "uniCOIL (noexp): otf" | |
| display-html: "uniCOIL (noexp): on-the-fly query inference" | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-noexp-0shot --topics $topics --encoder castorini/unicoil-noexp-msmarco-passage --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage | |
| topics: | |
| - topic_key: msmarco-v2-doc-dev | |
| eval_key: msmarco-v2-doc-dev | |
| scores: | |
| - MRR@100: 0.2232 | |
| R@1K: 0.8987 | |
| - topic_key: msmarco-v2-doc-dev2 | |
| eval_key: msmarco-v2-doc-dev2 | |
| scores: | |
| - MRR@100: 0.2314 | |
| R@1K: 0.8993 | |
| - topic_key: dl21 | |
| eval_key: dl21-doc | |
| scores: | |
| - MAP@100: 0.2589 | |
| nDCG@10: 0.6501 | |
| MRR@100: 0.9282 | |
| R@100: 0.3574 | |
| R@1K: 0.6782 | |
| - name: unicoil | |
| display: "uniCOIL (w/ doc2query-T5): pre-encoded" | |
| display-html: "uniCOIL (w/ doc2query-T5): pre-encoded queries" | |
| display-row: (3b) | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-0shot --topics $topics --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage | |
| topics: | |
| - topic_key: msmarco-v2-doc-dev-unicoil | |
| eval_key: msmarco-v2-doc-dev | |
| scores: | |
| - MRR@100: 0.2419 | |
| R@1K: 0.9122 | |
| - topic_key: msmarco-v2-doc-dev2-unicoil | |
| eval_key: msmarco-v2-doc-dev2 | |
| scores: | |
| - MRR@100: 0.2445 | |
| R@1K: 0.9172 | |
| - topic_key: dl21-unicoil | |
| eval_key: dl21-doc | |
| scores: | |
| - MAP@100: 0.2718 | |
| nDCG@10: 0.6783 | |
| MRR@100: 0.9684 | |
| R@100: 0.3700 | |
| R@1K: 0.7069 | |
| - name: unicoil-otf | |
| display: "uniCOIL (w/ doc2query-T5): otf" | |
| display-html: "uniCOIL (w/ doc2query-T5): on-the-fly query inference" | |
| command: python -m pyserini.search.lucene --threads 16 --batch-size 128 --index msmarco-v2-doc-segmented-unicoil-0shot --topics $topics --encoder castorini/unicoil-msmarco-passage --output $output --impact --hits 10000 --max-passage-hits 1000 --max-passage | |
| topics: | |
| - topic_key: msmarco-v2-doc-dev | |
| eval_key: msmarco-v2-doc-dev | |
| scores: | |
| - MRR@100: 0.2419 | |
| R@1K: 0.9120 | |
| - topic_key: msmarco-v2-doc-dev2 | |
| eval_key: msmarco-v2-doc-dev2 | |
| scores: | |
| - MRR@100: 0.2447 | |
| R@1K: 0.9174 | |
| - topic_key: dl21 | |
| eval_key: dl21-doc | |
| scores: | |
| - MAP@100: 0.2720 | |
| nDCG@10: 0.6782 | |
| MRR@100: 0.9684 | |
| R@100: 0.3702 | |
| R@1K: 0.7071 | |