Spaces:
Runtime error
Runtime error
File size: 5,561 Bytes
d6585f5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 |
#
# Pyserini: Reproducible IR research with sparse and dense representations
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os
from pyserini.index.lucene import IndexReader
from pyserini.util import compute_md5
beir_keys = {
'trec-covid': 'TREC-COVID',
'bioasq': 'BioASQ',
'nfcorpus': 'NFCorpus',
'nq': 'NQ',
'hotpotqa': 'HotpotQA',
'fiqa': 'FiQA-2018',
'signal1m': 'Signal-1M',
'trec-news': 'TREC-NEWS',
'robust04': 'Robust04',
'arguana': 'ArguAna',
'webis-touche2020': 'Webis-Touche2020',
'cqadupstack-android': 'CQADupStack-android',
'cqadupstack-english': 'CQADupStack-english',
'cqadupstack-gaming': 'CQADupStack-gaming',
'cqadupstack-gis': 'CQADupStack-gis',
'cqadupstack-mathematica': 'CQADupStack-mathematica',
'cqadupstack-physics': 'CQADupStack-physics',
'cqadupstack-programmers': 'CQADupStack-programmers',
'cqadupstack-stats': 'CQADupStack-stats',
'cqadupstack-tex': 'CQADupStack-tex',
'cqadupstack-unix': 'CQADupStack-unix',
'cqadupstack-webmasters': 'CQADupStack-webmasters',
'cqadupstack-wordpress': 'CQADupStack-wordpress',
'quora': 'Quora',
'dbpedia-entity': 'DBPedia',
'scidocs': 'SCIDOCS',
'fever': 'FEVER',
'climate-fever': 'Climate-FEVER',
'scifact': 'SciFact'
}
commitid = '505594'
date = '20221116'
type = 'flat'
for key in beir_keys:
index_reader = IndexReader(f'indexes/lucene-index.beir-v1.0.0-{key}.{type}.{date}.{commitid}')
stats = index_reader.stats()
md5 = compute_md5(f'indexes/lucene-index.beir-v1.0.0-{key}.{type}.{date}.{commitid}.tar.gz')
size = os.path.getsize(f'indexes/lucene-index.beir-v1.0.0-{key}.{type}.{date}.{commitid}.tar.gz')
print(f' "beir-v1.0.0-{key}.{type}": {{')
print(f' "description": "Lucene flat index of BEIR (v1.0.0): {beir_keys[key]}",')
print(f' "filename": "lucene-index.beir-v1.0.0-{key}.{type}.{date}.{commitid}.tar.gz",')
print(f' "readme": "lucene-index.beir-v1.0.0-{key}.{type}.{date}.{commitid}.README.md",')
print(f' "urls": [')
print(f' "https://rgw.cs.uwaterloo.ca/pyserini/indexes/lucene-index.beir-v1.0.0-{key}.{type}.{date}.{commitid}.tar.gz"')
print(f' ],')
print(f' "md5": "{md5}",')
print(f' "size compressed (bytes)": {size},')
print(f' "total_terms": {stats["total_terms"]},')
print(f' "documents": {stats["documents"]},')
print(f' "unique_terms": {stats["unique_terms"]},')
print(f' "downloaded": False')
print(f' }},')
# Stats for "contriever" indexes
for key in beir_keys:
index_reader = IndexReader(f'indexes/faiss.beir-v1.0.0-{key}.contriever.{date}.{commitid}')
stats = index_reader.stats()
md5 = compute_md5(f'indexes/faiss.beir-v1.0.0-{key}.contriever.{date}.{commitid}.tar.gz')
size = os.path.getsize(f'indexes/faiss.beir-v1.0.0-{key}.contriever.{date}.{commitid}.tar.gz')
print(f' "beir-v1.0.0-{key}.contriever": {{')
print(f' "description": "Faiss index for BEIR v1.0.0 ({beir_keys[key]}) corpus encoded by Contriever encoder.",')
print(f' "filename": "faiss.beir-v1.0.0-{key}.contriever.{date}.{commitid}.tar.gz",')
print(f' "readme": "faiss.beir-v1.0.0-{key}.contriever.{date}.{commitid}.README.md",')
print(f' "urls": [')
print(f' "https://rgw.cs.uwaterloo.ca/JIMMYLIN-bucket0/pyserini-indexes/faiss.beir-v1.0.0-{key}.contriever.{date}.{commitid}.tar.gz"')
print(f' ],')
print(f' "md5": "{md5}",')
print(f' "size compressed (bytes)": {size},')
print(f' "documents": {stats["documents"]},')
print(f' "downloaded": False,')
print(f' "texts": "beir-v1.0.0-{key}.flat"')
print(f' }},')
# Stats for "contriever" indexes with msmarco-ft
for key in beir_keys:
index_reader = IndexReader(f'indexes/faiss.beir-v1.0.0-{key}.contriever-msmarco.{date}')
stats = index_reader.stats()
md5 = compute_md5(f'indexes/faiss.beir-v1.0.0-{key}.contriever-msmarco.{date}.tar.gz')
size = os.path.getsize(f'indexes/faiss.beir-v1.0.0-{key}.contriever-msmarco.{date}.tar.gz')
print(f' "beir-v1.0.0-{key}.contriever": {{')
print(f' "description": "Faiss index for BEIR v1.0.0 ({beir_keys[key]}) corpus encoded by Contriever encoder fine-tuned with MS MARCO.",')
print(f' "filename": "faiss.beir-v1.0.0-{key}.contriever-msmarco.{date}.tar.gz",')
print(f' "readme": "faiss.beir-v1.0.0-{key}.contriever-msmarco.{date}.README.md",')
print(f' "urls": [')
print(f' "https://rgw.cs.uwaterloo.ca/pyserini/indexes/faiss.beir-v1.0.0-{key}.contriever-msmarco.20230124.tar.gz"')
print(f' ],')
print(f' "md5": "{md5}",')
print(f' "size compressed (bytes)": {size},')
print(f' "documents": {stats["documents"]},')
print(f' "downloaded": False,')
print(f' "texts": "beir-v1.0.0-{key}.flat"')
print(f' }},')
|