annotation ready

analysis.py

@@ -1,140 +0,0 @@
-import pandas as pd
-import numpy as np
-import os
-import torch
-from transformers import pipeline
-import streamlit as st
-
-import plotly.express as px
-import plotly.figure_factory as ff
-
-from captum.attr import LayerIntegratedGradients, TokenReferenceBase, visualization
-from captum.attr import visualization as viz
-from captum import attr
-from captum.attr._utils.visualization import format_word_importances, format_special_tokens, _get_color
-
-
-os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
-
-
-def results_to_df(results: dict, metric_name: str):
-    metric_scores = []
-    for topic, results_dict in results.items():
-        for metric_name_cur, metric_value in results_dict.items():
-            if metric_name == metric_name_cur:
-                metric_scores.append(metric_value)  
-    return pd.DataFrame({metric_name: metric_scores})
-
-
-def create_boxplot_1df(results: dict, metric_name: str):
-    df = results_to_df(results, metric_name)
-    fig = px.box(df, y=metric_name)
-    return fig
-
-
-def create_boxplot_2df(results1, results2, metric_name):
-    df1 = results_to_df(results1, metric_name)
-    df2 = results_to_df(results2, metric_name)
-    df2["Run"] = "Run 2"
-    df1["Run"] = "Run 1"
-    df = pd.concat([df1, df2])
-
-    # Create distplot with custom bin_size
-    fig = px.histogram(df, x=metric_name, color="Run", marginal="box", hover_data=df.columns)
-    return fig
-
-
-def create_boxplot_diff(results1, results2, metric_name):
-    df1 = results_to_df(results1, metric_name)
-    df2 = results_to_df(results2, metric_name)
-    diff = df1[metric_name] - df2[metric_name]
-
-    x_axis = f"Difference in {metric_name} from 1 to 2"
-    fig = px.histogram(pd.DataFrame({x_axis: diff}), x=x_axis, marginal="box")
-    return fig
-
-
-def summarize_attributions(attributions):
-    attributions = attributions.sum(dim=-1).squeeze(0)
-    attributions = attributions / torch.norm(attributions)
-    return attributions
-
-
-def get_words(words, importances):
-    words_colored = []
-    for word, importance in zip(words, importances[: len(words)]):
-        word = format_special_tokens(word)
-        color = _get_color(importance)
-        unwrapped_tag = '<span style="background-color: {color}; opacity:1.0; line-height:1.75">{word}</span>'.format(
-            color=color, word=word
-        )
-        words_colored.append(unwrapped_tag)
-    return words_colored
-
-@st.cache_resource
-def get_model(model_name: str):
-    if "MonoT5" in model_name:
-        if model_name == "MonoT5-Small":
-            pipe = pipeline('text2text-generation',
-                    model='castorini/monot5-small-msmarco-10k',
-                    tokenizer='castorini/monot5-small-msmarco-10k',
-                    device='cpu')
-        elif model_name == "MonoT5-3B":
-            pipe = pipeline('text2text-generation',
-                    model='castorini/monot5-3b-msmarco-10k',
-                    tokenizer='castorini/monot5-3b-msmarco-10k',
-                    device='cpu')
-        def formatter(query, doc):
-            return f"Query: {query} Document: {doc} Relevant:"
-    
-        
-    return pipe, formatter
-
-def prep_func(pipe, formatter):
-    # variables that only need to be run once
-    decoder_input_ids = pipe.tokenizer(["<pad>"], return_tensors="pt", add_special_tokens=False, truncation=True).input_ids.to('cpu')
-    decoder_embedding_layer = pipe.model.base_model.decoder.embed_tokens
-    decoder_inputs_emb = decoder_embedding_layer(decoder_input_ids)
-
-    token_false_id = pipe.tokenizer.get_vocab()['▁false']
-    token_true_id = pipe.tokenizer.get_vocab()["▁true"]
-
-    # this function needs to be run for each combination
-    @st.cache_data
-    def get_saliency(query, doc):
-        input_ids = pipe.tokenizer(
-                [formatter(query, doc)],
-                padding=False,
-                truncation=True,
-                return_tensors="pt",
-                max_length=pipe.tokenizer.model_max_length,
-        )["input_ids"].to('cpu')
-
-        embedding_layer = pipe.model.base_model.encoder.embed_tokens
-        inputs_emb = embedding_layer(input_ids)
-
-        def forward_from_embeddings(inputs_embeds, decoder_inputs_embeds):
-            logits = pipe.model.forward(inputs_embeds=inputs_embeds, decoder_inputs_embeds=decoder_inputs_embeds)['logits'][:, -1, :]
-            batch_scores = logits[:, [token_false_id, token_true_id]]
-            batch_scores = torch.nn.functional.log_softmax(batch_scores, dim=1)
-            scores = batch_scores[:, 1].exp() # relevant token
-            return scores
-        
-        lig = attr.Saliency(forward_from_embeddings)
-        attributions_ig, delta = lig.attribute(
-            inputs=(inputs_emb, decoder_inputs_emb)
-        )
-        attributions_normed = summarize_attributions(attributions_ig)
-        return "\n".join(get_words(pipe.tokenizer.convert_ids_to_tokens(input_ids.squeeze(0).tolist()), attributions_normed))
-        
-    return get_saliency
-
-
-if __name__ == "__main__":
-    query = "how to add dll to visual studio?"
-    doc = "StackOverflow In the days of 16-bit Windows, a WPARAM was a 16-bit word, while LPARAM was a 32-bit long. These distinctions went away in Win32; they both became 32-bit values. ... WPARAM is defined as UINT_PTR , which in 64-bit Windows is an unsigned, 64-bit value."
-    model, formatter = get_model("MonoT5")
-    get_saliency = prep_func(model, formatter)
-    print(get_saliency(query, doc))
-
-    

app.py

@@ -1,46 +1,58 @@
 import streamlit as st
 import os
 import pathlib
-import beir
-from beir import util
-from beir.datasets.data_loader import GenericDataLoader
-import pytrec_eval
 import pandas as pd
 from collections import defaultdict
 import json
 import copy
 import plotly.express as px
 
-from constants import ALL_DATASETS, ALL_METRICS
-from dataset_loading import get_dataset, load_run, load_local_qrels, load_local_corpus, load_local_queries
-from analysis import create_boxplot_1df, create_boxplot_2df, create_boxplot_diff, get_model, prep_func
+from dataset_loading import load_local_qrels, load_local_corpus, load_local_queries
 
 
 os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
 st.set_page_config(layout="wide")
 
+current_checkboxes = []
+query_input = None
 
-if 'cur_instance_num' not in st.session_state:
-    st.session_state.cur_instance_num = -1
+@st.cache_data
+def convert_df(df):
+    # IMPORTANT: Cache the conversion to prevent computation on every rerun
+    return df.to_csv(path_or_buf=None, index=False, quotechar='"').encode('utf-8')
+
+
+def create_histogram_relevant_docs(relevant_df):
+    # turn results into a dataframe and then plot
+    fig = px.histogram(relevant_df, x="relevant_docs")
+    # make it fit in one column
+    fig.update_layout(
+        height=400,
+        width=250
+    )
+    return fig
 
 
-def update_details(run_details, run_score):
-    if run_score == 0:
-        run_details["none"] += 1
-    elif run_score == 1:
-        run_details["perfect"] += 1
-    else:
-        run_details["inbetween"] += 1
-    return run_details
+def get_current_data():
+    cur_query_data = []
+    cur_query = query_input.replace("\n", "\\n")
+    for doc_id, checkbox in current_checkboxes:
+        if checkbox:
+            cur_query_data.append({
+                "new_narrative": cur_query,
+                "qid": st.session_state.selectbox_instance,
+                "doc_id": doc_id,
+                "is_relevant": 0
+            })
 
+    # return the data as a CSV pandas
+    return convert_df(pd.DataFrame(cur_query_data))
 
-def check_valid_args(run1_file, run2_file, dataset_name, qrels, queries, corpus):
-    if run1_file is not None and dataset_name not in ["", None, "custom"]:
-        return True
-    elif run1_file is not None and dataset_name == "custom":
-        if qrels is not None and queries is not None and corpus is not None:
-            return True
-    return False
+
+
+
+if 'cur_instance_num' not in st.session_state:
+    st.session_state.cur_instance_num = -1
 
 
 def validate(config_option, file_loaded):
@@ -49,196 +61,54 @@ def validate(config_option, file_loaded):
         st.stop()
 
 
-def combine(text_og, text_new, combine_type):
-    if combine_type == "None":
-        return text_og
-    elif combine_type == "Append":
-        return text_og + " <APPEND> " + text_new
-    elif combine_type == "Prepend":
-        return text_new + " <PREPEND> " + text_og
-    elif combine_type == "Replace":
-        return text_new
-    else:
-        raise ValueError("Invalid combine type")
-
 with st.sidebar:
     st.title("Options")
-    dataset_name = st.selectbox("Select a preloaded dataset or upload your own (note: some datasets are large/slow)", tuple(ALL_DATASETS))
-    if st.checkbox("Choose fields (applies to IR_Datasets only)"):
-        input_fields_doc = st.text_input("Type the name of the doc fields to get, with commas (blank=all)")
-        if input_fields_doc in ["", None]:
-            input_fields_doc = None
-        input_fields_query = st.sidebar.text_input("Type the name of the query fields to get, with commas (blank=all)")
-        if input_fields_query in ["", None]:
-            input_fields_query = None
-    else:
-        input_fields_doc = None
-        input_fields_query = None
-            
-    metric_name = st.selectbox("Select a metric", tuple(ALL_METRICS))
-
-    if dataset_name == "custom":
-        st.header("Upload corpus")
-        corpus_file = st.file_uploader("Choose a file", key="corpus")
-        corpus = load_local_corpus(corpus_file)
-        st.header("Upload queries")
-        queries_file = st.file_uploader("Choose a file", key="queries")
-        queries = load_local_queries(queries_file)
-        st.header("Upload qrels")
-        qrels_file = st.file_uploader("Choose a file", key="qrels")
-        qrels = load_local_qrels(qrels_file)
-    else:
-        qrels = None
-        queries = None
-        corpus = None
-
-
-    x = st.header('Upload a run file')
-    run1_file = st.file_uploader("Choose a file", key="run1")
-    y = st.header("Upload a second run file")
-    run2_file = st.file_uploader("Choose a file", key="run2")
+    st.header("Upload corpus")
+    corpus_file = st.file_uploader("Choose a file", key="corpus")
+    corpus = load_local_corpus(corpus_file)
+    st.header("Upload queries")
+    queries_file = st.file_uploader("Choose a file", key="queries")
+    queries = load_local_queries(queries_file)
+    st.header("Upload qrels")
+    qrels_file = st.file_uploader("Choose a file", key="qrels")
+    qrels = load_local_qrels(qrels_file)
+
+    ## make sure all qids in qrels are in queries and write out a warning if not
+    if queries is not None and qrels is not None:
+        missing_qids = set(qrels.keys()) - set(queries.keys()) | set(queries.keys()) - set(qrels.keys())
+        if len(missing_qids) > 0:
+            st.warning(f"The following qids in qrels are not in queries and will be deleted: {missing_qids}")
+            # remove them from qrels and queries
+            for qid in missing_qids:
+                if qid in qrels:
+                    del qrels[qid]
+                if qid in queries:
+                    del queries[qid]
+
+    data = []
+    for key, value in qrels.items():
+        data.append({"relevant_docs": len(value)})
+    relevant_df = pd.DataFrame(data)
 
     z = st.header("Analysis Options")
     # sliderbar of how many Top N to choose
-    top_n = st.slider("Top N Ranked Docs", 1, 100, 3)
-    n_relevant_docs = st.slider("Number of relevant docs", 1, 100, 3)
-    incorrect_only = st.checkbox("Show only incorrect instances", value=False)
-    one_better_than_two = st.checkbox("Show only instances where run 1 is better than run 2", value=False)
-    two_better_than_one = st.checkbox("Show only instances where run 2 is better than run 1", value=False)
-    use_model_saliency = st.checkbox("Use model saliency (slow!)", value=False)
-    if use_model_saliency:
-        # choose from a list of models
-        model_name = st.selectbox("Choose from a list of models", ["MonoT5-Small", "MonoT5-3B"])
-        model, formatter = get_model(model_name)
-        get_saliency = prep_func(model, formatter)
-
-
-    advanced_options1 = st.checkbox("Show advanced options for Run 1", value=False)
-    doc_expansion1 = doc_expansion2 = None
-    query_expansion1 = query_expansion2 = None
-    run1_uses_query_expansion = "None"
-    run1_uses_doc_expansion = "None"
-    run2_uses_query_expansion = "None"
-    run2_uses_doc_expansion = "None"
-    if advanced_options1:
-        doc_header = st.header("Upload a Document Expansion file")
-        doc_expansion_file = st.file_uploader("Choose a file", key="doc_expansion")
-        if doc_expansion_file is not None:
-            doc_expansion1 = load_local_corpus(doc_expansion_file)
-        query_header = st.header("Upload a Query Expansion file")
-        query_expansion_file = st.file_uploader("Choose a file", key="query_expansion")
-        if query_expansion_file is not None:
-            query_expansion1 = load_local_queries(query_expansion_file)
-
-        run1_uses_query_expansion = st.selectbox("Type of query expansion used in run 1", ("None", "Append", "Prepend", "Replace"))
-        run1_uses_doc_expansion = st.selectbox("Type of document expansion used in run 1", ("None", "Append", "Prepend", "Replace"))
-        validate(run1_uses_query_expansion, query_expansion_file)
-        validate(run1_uses_doc_expansion, doc_expansion_file)
-
-    advanced_options2 = st.checkbox("Show advanced options for Run 2", value=False)
-    if advanced_options2:
-        doc_header = st.header("Upload a Document Expansion file")
-        doc_expansion_file = st.file_uploader("Choose a file", key="doc_expansion2")
-        if doc_expansion_file is not None:
-            doc_expansion2 = load_local_corpus(doc_expansion_file)
-        query_header = st.header("Upload a Query Expansion file")
-        query_expansion_file = st.file_uploader("Choose a file", key="query_expansion2")
-        if query_expansion_file is not None:
-            query_expansion2 = load_local_queries(query_expansion_file)
-
-        run2_uses_query_expansion = st.selectbox("Type of query expansion used in run 2", ("None", "Append", "Prepend", "Replace"))
-        run2_uses_doc_expansion = st.selectbox("Type of document expansion used in run 2", ("None", "Append", "Prepend", "Replace"))
-        validate(run2_uses_query_expansion, query_expansion_file)
-        validate(run2_uses_doc_expansion, doc_expansion_file)
-
-
-# everything hinges on the run being uploaded, so do that first
-# init_title = st.title("Upload Run and Choose Details")
-
-if run1_file is not None:
-    run1, run1_pandas = load_run(run1_file)
-
-# do everything, now that we have the run file
-if check_valid_args(run1_file, run2_file, dataset_name, qrels, queries, corpus):
-    # init_title = st.title("Analysis")
-    # don't load these til a run is given
-    if dataset_name != "custom":
-        corpus, queries, qrels = get_dataset(dataset_name, input_fields_doc, input_fields_query)
-
-    evaluator = pytrec_eval.RelevanceEvaluator(
-            copy.deepcopy(qrels), pytrec_eval.supported_measures)
-    results1 = evaluator.evaluate(run1) # dict of instance then metrics then values
-    average_run1_score = pytrec_eval.compute_aggregated_measure(metric_name, [query_measures[metric_name] for query_measures in results1.values()])
-    if len(results1) == 0:
-        # alert and stop
-        st.error("Run file is empty")
-        st.stop()
+    n_relevant_docs = st.slider("Number of relevant docs", 1, 999, 20)
 
-    if run2_file is not None:
-        run2, run2_pandas = load_run(run2_file)
-        # NOTE: will fail if run1 is not uploaded
-        evaluator2 = pytrec_eval.RelevanceEvaluator(
-            copy.deepcopy(qrels), pytrec_eval.supported_measures)
-        results2 = evaluator2.evaluate(run2)
-        average_run2_score = pytrec_eval.compute_aggregated_measure(metric_name, [query_measures[metric_name] for query_measures in results2.values()])
 
+col1, col2 = st.columns([1, 3], gap="large")
 
-    col1, col2 = st.columns([1, 3], gap="large")
+if corpus is not None and queries is not None and qrels is not None:
+    with st.sidebar:
+        st.success("All files uploaded")
 
-    # incorrect = 0
-    is_better_run1_count = 0
-    is_better_run2_count = 0
-    is_same_count = 0
-    run1_details = {"none": 0, "perfect": 0, "inbetween": 0}
-    run2_details = {"none": 0, "perfect": 0, "inbetween": 0}
     with col1:
+        # breakpoint()
+        set_of_cols =  set(qrels.keys())
+        container_for_nav = st.container()
+        name_of_columns = sorted([item for item in set_of_cols])
+        instances_to_use = name_of_columns
         st.title("Instances")
-        if run1_file is not None:
-            set_of_cols =  set(run1_pandas.qid.tolist())
-            container_for_nav = st.container()
-            name_of_columns = sorted([item for item in set_of_cols])
-            instances_to_use = []
-            # st.divider()
-            for idx in range(len(name_of_columns)):
-                is_incorrect = False
-                is_better_run1 = False
-                is_better_run2 = False
-
-                run1_score = results1[str(name_of_columns[idx])][metric_name] if idx else 1
-                run1_details = update_details(run1_details, run1_score)
-                if run2_file is not None:
-                    run2_score = results2[str(name_of_columns[idx])][metric_name] if idx else 1
-                    run2_details = update_details(run2_details, run2_score)
-
-                    if run1_score == 0 or run2_score == 0:
-                        is_incorrect = True
-
-                    if run1_score > run2_score:
-                        is_better_run1_count += 1
-                        is_better_run1 = True
-                    elif run2_score > run1_score:
-                        is_better_run2_count += 1
-                        is_better_run2 = True
-                    else:
-                        is_same_count += 1
-
-
-                    if not incorrect_only or is_incorrect:
-                        if not one_better_than_two or is_better_run1:
-                            if not two_better_than_one or is_better_run2:
-                                # check = st.checkbox(f"{idx}. " + str(name_of_columns[idx]), key=f"{idx}check")  
-                                # st.divider()
-                                instances_to_use.append(name_of_columns[idx])
-                else:
-                    if run1_score == 0:
-                        is_incorrect = True
-
-                    if not incorrect_only or is_incorrect:
-                        # check = st.checkbox(f"{idx}. " + str(name_of_columns[idx]), key=f"{idx}check")  
-                        # st.divider()
-                        instances_to_use.append(name_of_columns[idx])
-
-
+        
         def sync_from_drop():
             if st.session_state.selectbox_instance == "Overview":
                 st.session_state.number_of_col = -1
@@ -261,378 +131,62 @@ if check_valid_args(run1_file, run2_file, dataset_name, qrels, queries, corpus):
         number_of_col = container_for_nav.number_input(min_value=-1, step=1, max_value=len(instances_to_use) - 1, on_change=sync_from_number, label=f"Select instance by index (up to **{len(instances_to_use) - 1}**)", key="number_of_col")
         selectbox_instance = container_for_nav.selectbox("Select instance by ID", ["Overview"] + name_of_columns, on_change=sync_from_drop, key="selectbox_instance")
         st.divider()  
-        # make pie plot showing incorrect vs correct
-        st.header("Breakdown")
-
-
-        if run2_file is None:
-            overall_scores_container = st.container()
-            left_score, right_score = overall_scores_container.columns([1, 1])
-            left_score.metric(label=f"Run 1 {metric_name}", value=round(average_run1_score, 3))
-            right_score.metric(label="#Q", value=len(results1))
-
-            plotly_pie_chart = px.pie(names=["Perfect", "Inbetween", "None"], values=[run1_details["perfect"], run1_details["inbetween"], run1_details["none"]])
-            st.write("Run 1 Scores")
-            plotly_pie_chart.update_traces(showlegend=False, selector=dict(type='pie'), textposition='inside', textinfo='percent+label')
-            st.plotly_chart(plotly_pie_chart, use_container_width=True)
-        else:
-            overall_scores_container = st.container()
-            left_score, right_score = overall_scores_container.columns([1, 1])
-            left_score.metric(label=f"Run 1 {metric_name}", value=round(average_run1_score, 3))
-            right_score.metric(label=f"Run 2 {metric_name}", value=round(average_run2_score, 3))
-
-            if st.checkbox("Show Run 1 vs Run 2", value=True):
-                plotly_pie_chart = px.pie(names=["Run 1 Better", "Run 2 Better", "Tied"], values=[is_better_run1_count, is_better_run2_count, is_same_count])
-                plotly_pie_chart.update_traces(showlegend=False, selector=dict(type='pie'), textposition='inside', textinfo='percent+label')
-                st.plotly_chart(plotly_pie_chart, use_container_width=True)
-                
-            if st.checkbox("Show Run 1 Breakdown"):
-                plotly_pie_chart_run1 = px.pie(names=["Perfect", "Inbetween", "None"], values=[run1_details["perfect"], run1_details["inbetween"], run1_details["none"]])
-                plotly_pie_chart_run1.update_traces(showlegend=False, selector=dict(type='pie'), textposition='inside', textinfo='percent+label')
-                st.plotly_chart(plotly_pie_chart_run1, use_container_width=True)
-            if st.checkbox("Show Run 2 Breakdown"):
-                plotly_pie_chart_run2 = px.pie(names=["Perfect", "Inbetween", "None"], values=[run2_details["perfect"], run2_details["inbetween"], run2_details["none"]])
-                plotly_pie_chart_run2.update_traces(showlegend=False, selector=dict(type='pie'), textposition='inside', textinfo='percent+label')
-                st.plotly_chart(plotly_pie_chart_run2, use_container_width=True)
-
+        # make pie plot showing how many relevant docs there are per query histogram
+        st.header("Relevant Docs Per Query")
+        plotly_chart = create_histogram_relevant_docs(relevant_df)
+        st.plotly_chart(plotly_chart)
+        st.divider()
+        # now show the number with relevant docs less than `n_relevant_docs`
+        st.header("Relevant Docs Less Than {}:".format(n_relevant_docs))
+        st.subheader(f'{relevant_df[relevant_df["relevant_docs"] < n_relevant_docs].shape[0]} Queries')
 
 
     with col2:
-        # st.title(f"Information ({len(checkboxes) - 1}/{len(name_of_columns) - 1})")
-        ### Only one run file
-        if run1_file is not None and run2_file is None:
-
-            # get instance number
-            inst_index = number_of_col
-
-            if inst_index >= 0:
-                inst_num = instances_to_use[inst_index - 1]
-                
-                st.markdown("<h1 style='text-align: center; color: black;text-decoration: underline;'>Run 1</h1>", unsafe_allow_html=True)
-
-                container = st.container()
-
-                rank_col, score_col, id_col = container.columns([2,1,3])
-                id_col.metric("ID", inst_num)
-                score_col.metric(metric_name, results1[str(inst_num)][metric_name])
-
-                # st.subheader(f"ID")
-                # st.markdown(inst_num)
-                st.divider()
-
-                st.subheader(f"Query")
-                if run1_uses_query_expansion != "None":
-                    show_orig_rel = st.checkbox("Show Original Query", key=f"{inst_index}reloriguery", value=False)
-
-                query_text_og = queries[str(inst_num)]
-                if query_expansion1 is not None and run1_uses_query_expansion != "None" and not show_orig_rel:
-                    alt_text = query_expansion1[str(inst_num)]
-                    query_text = combine(query_text_og, alt_text, run1_uses_query_expansion)
-                else:
-                    query_text = query_text_og
-                st.markdown(query_text)
-                st.divider()
-
-                ## Documents
-                # relevant
-                relevant_docs = list(qrels[str(inst_num)].keys())[:n_relevant_docs]
-                doc_texts = [(doc_id, corpus[doc_id]["title"] if "title" in corpus[doc_id] else "", corpus[doc_id]["text"]) for doc_id in relevant_docs]
-                st.subheader("Relevant Documents")
-                if doc_expansion1 is not None and run1_uses_doc_expansion != "None":
-                    show_orig_rel = st.checkbox("Show Original Relevant Doc(s)", key=f"{inst_index}relorig", value=False)
-
-                for (docid, title, text) in doc_texts:
-                    if doc_expansion1 is not None and run1_uses_doc_expansion != "None" and not show_orig_rel:
-                        alt_text = doc_expansion1[docid]["text"]
-                        text = combine(text, alt_text, run1_uses_doc_expansion)
-
-                    if use_model_saliency:
-                        if st.checkbox("Show Model Saliency", key=f"{inst_index}model_saliency", value=False):
-                            st.markdown(get_saliency(query_text, doc_texts),unsafe_allow_html=True)
-                        else:
-                            st.text_area(f"{docid}:", text)
-
-                    else:
-                        st.text_area(f"{docid}:", text)
-
-
-                # go through each of the relevant documents
-                ranks = []
-                for docid in relevant_docs:
-                    pred_doc = run1_pandas[run1_pandas.doc_id.isin([docid])]
-                    rank_pred = pred_doc[pred_doc.qid == str(inst_num)]
-                    if rank_pred.empty:
-                        ranks.append("-")
-                    else:
-                        ranks.append(rank_pred.iloc[0]["rank"])
-                # st.subheader("Ranked of Documents")
-                # st.markdown(f"Rank: {rank_pred}")
-                ranking_str = ",".join([str(item) for item in ranks])
-                if ranking_str == "":
-                    ranking_str = "-"
-                rank_col.metric(f"Rank of Relevant Doc(s)", ranking_str)
-                # breakpoint()
-
-
-                st.divider()
-
-                # top ranked
-
-                if st.checkbox('Show top ranked documents', key=f"{inst_index}top-1run"):
-                    st.subheader("Top N Ranked Documents")
-                    if doc_expansion1 is not None and run1_uses_doc_expansion != "None":
-                        show_orig_rel_ranked = st.checkbox("Show Original Ranked Doc(s)", key=f"{inst_index}relorigdocs", value=False)
-
-                    run1_top_n = run1_pandas[run1_pandas.qid == str(inst_num)][:top_n]
-                    run1_top_n_docs = [corpus[str(doc_id)] for doc_id in run1_top_n.doc_id.tolist()]
-                    if doc_expansion1 is not None and run1_uses_doc_expansion != "None" and not show_orig_rel_ranked:
-                        run1_top_n_docs_alt = [doc_expansion1[str(doc_id)] for doc_id in run1_top_n.doc_id.tolist()]
-                        for d_idx, doc in enumerate(run1_top_n_docs):
-                            alt_text = run1_top_n_docs_alt[d_idx]["text"]
-                            doc_text = combine(doc["text"], alt_text, run1_uses_doc_expansion)
-                            if use_model_saliency:
-                                if st.checkbox("Show Model Saliency", key=f"{inst_index}model_saliency", value=False):
-                                    st.markdown(get_saliency(query_text, doc_text),unsafe_allow_html=True)
-                                else:
-                                    st.text_area(f"{run1_top_n['doc_id'].iloc[d_idx]}: ", doc_text, key=f"{inst_num}doc{d_idx}")
-                            else:
-                                st.text_area(f"{run1_top_n['doc_id'].iloc[d_idx]}: ", doc_text, key=f"{inst_num}doc{d_idx}")
-                    else:
-                        for d_idx, doc in enumerate(run1_top_n_docs):
-                            if use_model_saliency:
-                                if st.checkbox("Show Model Saliency", key=f"{inst_index}model_saliency{d_idx}ranked", value=False):
-                                    st.markdown(get_saliency(query_text, doc),unsafe_allow_html=True)
-                                else:
-                                    st.text_area(f"{run1_top_n['doc_id'].iloc[d_idx]}: ", doc["text"], key=f"{inst_num}doc{d_idx}")
-                            else:
-                                st.text_area(f"{run1_top_n['doc_id'].iloc[d_idx]}: ", doc["text"], key=f"{inst_num}doc{d_idx}")
-                    st.divider()
-        
-            # none checked
-            elif inst_index < 0:
-                st.title("Overview")
-                st.subheader(f"Scores of {metric_name}")
-                plotly_chart = create_boxplot_1df(results1, metric_name)
-                st.plotly_chart(plotly_chart)
-                                
-        ## Both run files available
-        elif run1_file is not None and run2_file is not None:
-            has_check = False
-            container_top = st.container()
-
-            # get instance number
-            inst_index = number_of_col
-           
-            if inst_index >= 0:
-                inst_num = instances_to_use[inst_index]
-    
-                col_run1, col_run2 = container_top.columns([1,1])
-                col_run1.markdown("<h1 style='text-align: center; color: black;text-decoration: underline;'>Run 1</h1>", unsafe_allow_html=True)
-                col_run2.markdown("<h1 style='text-align: center; color: black;text-decoration: underline;'>Run 2</h1>", unsafe_allow_html=True)
-
-                container_overview = st.container()
-                rank_col1, score_col1, rank_col2, score_col2  = container_overview.columns([2,1,2,1])
-                # id_col1.metric("", "")
-                score_col1.metric("Run 1 " + metric_name, results1[str(inst_num)][metric_name])
-                score_col2.metric("Run 2 " + metric_name, results2[str(inst_num)][metric_name])
-
-                st.divider()
-
-                st.subheader(f"Query")
-                container_two_query = st.container()
-                col_run1, col_run2 = container_two_query.columns(2, gap="medium")
-
-                query_text_og = queries[str(inst_num)]
-                if run1_uses_query_expansion != "None" and run2_uses_query_expansion != "None":
-                    alt_text1 = query_expansion1[str(inst_num)]
-                    alt_text2 = query_expansion2[str(inst_num)]
-                    combined_text1 = combine(query_text_og, alt_text1, run1_uses_query_expansion)
-                    combined_text2 = combine(query_text_og, alt_text2, run2_uses_query_expansion)
-                    col_run1.markdown(combined_text1)
-                    col_run2.markdown(combined_text2)
-                    query_text1 = combined_text1
-                    query_text2 = combined_text2
-                elif run1_uses_query_expansion != "None":
-                    alt_text = query_expansion1[str(inst_num)]
-                    combined_text1 = combine(query_text_og, alt_text, run1_uses_query_expansion)
-                    col_run1.markdown(combined_text1)
-                    col_run2.markdown(query_text_og)
-                    query_text1 = combined_text1
-                    query_text2 = query_text_og
-                elif run2_uses_query_expansion != "None":
-                    alt_text = query_expansion2[str(inst_num)]
-                    combined_text2 = combine(query_text_og, alt_text, run2_uses_query_expansion)
-                    col_run1.markdown(query_text_og)
-                    col_run2.markdown(combined_text2)
-                    query_text1 = query_text_og
-                    query_text2 = combined_text2
-                else:
-                    query_text = query_text_og
-                    col_run1.markdown(query_text)
-                    col_run2.markdown(query_text)
-                    query_text1 = query_text
-                    query_text2 = query_text
-
-                st.divider()
-
-
-
-                ## Documents
-                # relevant
-                st.subheader("Relevant Documents")
-                container_two_docs_rel = st.container()
-                col_run1, col_run2 = container_two_docs_rel.columns(2, gap="medium")
-                relevant_docs = list(qrels[str(inst_num)].keys())[:n_relevant_docs]
-                relevant_score = {ind_doc_id: qrels[str(inst_num)][ind_doc_id] for ind_doc_id in relevant_docs}
-                doc_texts = [(doc_id, corpus[doc_id]["title"] if "title" in corpus[doc_id] else "", corpus[doc_id]["text"], relevant_score[doc_id]) for doc_id in relevant_docs]
-
-                if doc_expansion1 is not None and run1_uses_doc_expansion != "None":
-                    show_orig_rel1 = col_run1.checkbox("Show Original Relevant Doc(s)", key=f"{inst_index}relorig_run1", value=False)
-                if doc_expansion2 is not None and run2_uses_doc_expansion != "None":
-                    show_orig_rel2 = col_run2.checkbox("Show Original Relevant Doc(s)", key=f"{inst_index}relorig_run2", value=False)
-
-                for (docid, title, text, rel_score) in doc_texts:
-                    if doc_expansion1 is not None and run1_uses_doc_expansion != "None" and not show_orig_rel1:
-                        alt_text = doc_expansion1[docid]["text"]
-                        text = combine(text, alt_text, run1_uses_doc_expansion)
-                    
-                    if use_model_saliency:
-                        if col_run1.checkbox("Show Model Saliency", key=f"{inst_index}model_saliency{docid}relevant", value=False):
-                            col_run1.markdown(get_saliency(query_text1, text),unsafe_allow_html=True)
-                        else:
-                            col_run1.text_area(f"{docid} (Rel: {rel_score}):", text, key=f"{inst_num}doc{docid}1")
-                    else:
-                        col_run1.text_area(f"{docid} (Rel: {rel_score}):", text, key=f"{inst_num}doc{docid}1")
-
-                for (docid, title, text, rel_score) in doc_texts:
-                    if doc_expansion2 is not None and run2_uses_doc_expansion != "None" and not show_orig_rel2:
-                        alt_text = doc_expansion2[docid]["text"] if docid in doc_expansion2 else "<NOT EXPANDED>"
-                        text = combine(text, alt_text, run2_uses_doc_expansion)
-
-                    if use_model_saliency:
-                        if col_run2.checkbox("Show Model Saliency", key=f"{inst_index}model_saliency{docid}relevant2", value=False):
-                            col_run2.markdown(get_saliency(query_text2, text),unsafe_allow_html=True)
-                        else:
-                            col_run2.text_area(f"{docid}: (Rel: {rel_score})", text, key=f"{inst_num}doc{docid}2")
-                    else:
-                        col_run2.text_area(f"{docid}: (Rel: {rel_score})", text, key=f"{inst_num}doc{docid}2")
-
-                # top ranked
-                # NOTE: BEIR calls trec_eval which ranks by score, then doc_id for ties
-                # we have to fix that or we don't match the scores
-
-                ranks2 = []
-                for docid in relevant_docs:
-                    pred_doc = run2_pandas[run2_pandas.doc_id.isin([docid])]
-                    rank_pred = pred_doc[pred_doc.qid == str(inst_num)]
-                    if rank_pred.empty:
-                        ranks2.append("-")
-                    else:
-                        ranks2.append(rank_pred.iloc[0]["rank"])
-                # st.subheader("Ranked of Documents")
-                # st.markdown(f"Rank: {rank_pred}")
-                ranking_str2 = ",".join([str(item) for item in ranks2])
-                if ranking_str2 == "":
-                    ranking_str2 = "-"
-                rank_col2.metric("Run 2 " + f"Rank of Relevant Doc(s)", ranking_str2)
-
-
-                ranks1 = []
-                for docid in relevant_docs:
-                    pred_doc = run1_pandas[run1_pandas.doc_id.isin([docid])]
-                    rank_pred = pred_doc[pred_doc.qid == str(inst_num)]
-                    if rank_pred.empty:
-                        ranks1.append("-")
-                    else:
-                        ranks1.append(rank_pred.iloc[0]["rank"])
-                # st.subheader("Ranked of Documents")
-                # st.markdown(f"Rank: {rank_pred}")
-                ranking_str1 = ",".join([str(item) for item in ranks1])
-                if ranking_str1 == "":
-                    ranking_str1 = "-"
-                rank_col1.metric("Run 1 " + f"Rank of Relevant Doc(s)", ranking_str1)
-
-
-                st.divider()
-
-
-                container_two_docs_ranked = st.container()
-                col_run1, col_run2 = container_two_docs_ranked.columns(2, gap="medium")
-
-                if col_run1.checkbox('Show top ranked documents for Run 1', key=f"{inst_index}top-1run"):
-                    col_run1.subheader("Top N Ranked Documents")
-                    if doc_expansion1 is not None and run1_uses_doc_expansion != "None":
-                        show_orig_rel_ranked1 = col_run1.checkbox("Show Original Ranked Doc(s)", key=f"{inst_index}relorigdocs1", value=False)
-
-                    run1_top_n = run1_pandas[run1_pandas.qid == str(inst_num)].sort_values(["score", "doc_id"], ascending=[False, False])[:top_n]
-                    run1_top_n_docs = [corpus[str(doc_id)] for doc_id in run1_top_n.doc_id.tolist()]
-
-                    if doc_expansion1 is not None and run1_uses_doc_expansion != "None" and not show_orig_rel_ranked1:
-                        run1_top_n_docs_alt = [doc_expansion1[str(doc_id)] for doc_id in run1_top_n.doc_id.tolist()]
-                        for d_idx, doc in enumerate(run1_top_n_docs):
-                            alt_text = run1_top_n_docs_alt[d_idx]["text"]
-                            doc_text = combine(doc["text"], alt_text, run1_uses_doc_expansion)
-                            if use_model_saliency:
-                                if col_run1.checkbox("Show Model Saliency", key=f"{inst_index}model_saliency{d_idx}ranked1", value=False):
-                                    col_run1.markdown(get_saliency(query_text1, doc_text),unsafe_allow_html=True)
-                                else:
-                                    col_run1.text_area(f"{run1_top_n['doc_id'].iloc[d_idx]}: ", doc_text, key=f"{inst_num}doc{d_idx}1")
-                            else:
-                                col_run1.text_area(f"{run1_top_n['doc_id'].iloc[d_idx]}: ", doc_text, key=f"{inst_num}doc{d_idx}1")
-                    else:
-                        for d_idx, doc in enumerate(run1_top_n_docs):
-                            if use_model_saliency:
-                                if col_run1.checkbox("Show Model Saliency", key=f"{inst_index}model_saliency{d_idx}ranked1", value=False):
-                                    col_run1.markdown(get_saliency(query_text1, doc),unsafe_allow_html=True)
-                                else:
-                                    col_run1.text_area(f"{run1_top_n['doc_id'].iloc[d_idx]}: ", doc["text"], key=f"{inst_num}doc{d_idx}1")
-                            else:
-                                col_run1.text_area(f"{run1_top_n['doc_id'].iloc[d_idx]}: ", doc["text"], key=f"{inst_num}doc{d_idx}1")
-                                
-                    
-                if col_run2.checkbox('Show top ranked documents for Run 2', key=f"{inst_index}top-2run"):
-                    col_run2.subheader("Top N Ranked Documents")
-                    if doc_expansion2 is not None and run2_uses_doc_expansion != "None":
-                        show_orig_rel_ranked2 = col_run2.checkbox("Show Original Ranked Doc(s)", key=f"{inst_index}relorigdocs2", value=False)        
-                    run2_top_n = run2_pandas[run2_pandas.qid == str(inst_num)].sort_values(["score", "doc_id"], ascending=[False, False])[:top_n]
-                    run2_top_n_docs = [corpus[str(doc_id)] for doc_id in run2_top_n.doc_id.tolist()]
-
-
-                    if doc_expansion2 is not None and run2_uses_doc_expansion != "None" and not show_orig_rel_ranked2:
-                        run2_top_n_docs_alt = [doc_expansion2[str(doc_id)] for doc_id in run2_top_n.doc_id.tolist()]
-                        for d_idx, doc in enumerate(run2_top_n_docs):
-                            alt_text = run2_top_n_docs_alt[d_idx]["text"]
-                            doc_text = combine(doc["text"], alt_text, run2_uses_doc_expansion)
-                            if use_model_saliency:
-                                if col_run2.checkbox("Show Model Saliency", key=f"{inst_index}model_saliency{d_idx}ranked2", value=False):
-                                    col_run2.markdown(get_saliency(query_text2, doc_text),unsafe_allow_html=True)
-                                else:
-                                    col_run2.text_area(f"{run2_top_n['doc_id'].iloc[d_idx]}: ", doc_text, key=f"{inst_num}doc{d_idx}2")
-                            else:
-                                col_run2.text_area(f"{run2_top_n['doc_id'].iloc[d_idx]}: ", doc_text, key=f"{inst_num}doc{d_idx}2")
-                    else:
-                        for d_idx, doc in enumerate(run2_top_n_docs):
-                            if use_model_saliency:
-                                if col_run2.checkbox("Show Model Saliency", key=f"{inst_index}model_saliency{d_idx}ranked2", value=False):
-                                    col_run2.markdown(get_saliency(query_text2, doc),unsafe_allow_html=True)
-                                else:
-                                    col_run2.text_area(f"{run2_top_n['doc_id'].iloc[d_idx]}: ", doc["text"], key=f"{inst_num}doc{d_idx}2")
-                            else:
-                                col_run2.text_area(f"{run2_top_n['doc_id'].iloc[d_idx]}: ", doc["text"], key=f"{inst_num}doc{d_idx}2")
-
-                st.divider()
+        # get instance number
+        inst_index = number_of_col
+
+        if inst_index >= 0:
+            inst_num = instances_to_use[inst_index]
+            
+            st.markdown("<h1 style='text-align: center; color: black;text-decoration: underline;'>Editor</h1>", unsafe_allow_html=True)
 
 
-            else:
-                st.title("Overview")
+            container = st.container()
+
 
-                st.subheader(f"Scores of {metric_name}")
-                fig = create_boxplot_2df(results1, results2, metric_name)
-                st.plotly_chart(fig)
+            container.divider()
 
-                st.subheader(f"Score Difference of {metric_name}")
-                fig_comp = create_boxplot_diff(results1, results2, metric_name)
-                st.plotly_chart(fig_comp)
+            container.subheader(f"Query")
+            
+            query_text = queries[str(inst_num)].strip()
+            query_input = container.text_area(f"QID: {inst_num}", query_text)
+            container.divider()
+
+            ## Documents
+            # relevant
+            relevant_docs = list(qrels[str(inst_num)].keys())[:n_relevant_docs]
+            doc_texts = [(doc_id, corpus[doc_id]["title"] if "title" in corpus[doc_id] else "", corpus[doc_id]["text"]) for doc_id in relevant_docs]
+            container.subheader(f"Relevant Documents ({len(list(qrels[str(inst_num)].keys()))})")
+            current_checkboxes = []
+            for (docid, title, text) in doc_texts:
+                current_checkboxes.append((docid, container.checkbox(f'{docid} is Non-Relevant', key=docid)))
+                container.text_area(f"{docid}:", text)
+
+
+            container.divider()
+            if st.checkbox("Download data as CSV"):
+                st.download_button(
+                    label="Download data as CSV",
+                    data=get_current_data(),
+                    file_name=f'annotation_query_{inst_num}.csv',
+                    mime='text/csv',
+                )
+
+        # none checked
+        elif inst_index < 0:
+            st.title("Overview")
+
+    
 
 else:
     st.warning("Please choose a dataset and upload a run file. If you chose \"custom\" be sure that you uploaded all files (queries, corpus, qrels)")

constants.py

@@ -1,90 +0,0 @@
-from ir_dataset_metadata import IR_DATASETS
-
-
-ALL_METRICS = [
-    "ndcg_cut_10",
-    "ndcg_cut_5",
-    "ndcg_cut_15",
-    "ndcg_cut_20", 
-    "ndcg_cut_30", 
-    "ndcg_cut_100", 
-    "ndcg_cut_200", 
-    "ndcg_cut_500", 
-    "ndcg_cut_1000", 
-    "map", 
-    "P_5", 
-    "P_10", 
-    "P_15", 
-    "P_20", 
-    "P_30", 
-    "P_100", 
-    "P_200", 
-    "P_500", 
-    "P_1000", 
-    "recall_5", 
-    "recall_10", 
-    "recall_15", 
-    "recall_20", 
-    "recall_30", 
-    "recall_100", 
-    "recall_200", 
-    "recall_500", 
-    "recall_1000", 
-    "recip_rank", 
-    "set_recall", 
-    "set_P", 
-    "set_F", 
-    "num_rel_ret", 
-    "num_ret", 
-    "num_rel", 
-    "num_q", 
-    "num_rel", 
-    "num_rel_ret"
-    "Rprec",
-    "bpref",
-    "iprec_at_recall_0.00",
-    "iprec_at_recall_0.10",
-    "iprec_at_recall_0.20",
-    "iprec_at_recall_0.30",
-    "iprec_at_recall_0.40",
-    "iprec_at_recall_0.50",
-    "iprec_at_recall_0.60",
-    "iprec_at_recall_0.70",
-    "iprec_at_recall_0.80",
-    "iprec_at_recall_0.90",
-    "iprec_at_recall_1.00",
-]
-
-
-BEIR = [
-    "msmarco",
-    "trec-covid",
-    "nf_corpus",
-    "bioasq",
-    "nq",
-    "hotpotqa",
-    "fiqa",
-    "signal1m",
-    "trec-news",
-    "robust04",
-    "arguana",
-    "webis-touche2020",
-    "cqadupstack",
-    "quora",
-    "dbpedia-entity",
-    "scidocs",
-    "fever",
-    "climate-fever",
-    "scifact",
-]
-
-
-LOCAL_DATASETS = [
-    "gooaq_technical",
-    "codesearch_py",
-]
-
-
-    
-    
-ALL_DATASETS = ["", "custom"] + LOCAL_DATASETS + BEIR + IR_DATASETS

dataset_loading.py

@@ -1,19 +1,13 @@
 import streamlit as st
 import os
 import pathlib
-import beir
-from beir import util
-from beir.datasets.data_loader import GenericDataLoader
-import pytrec_eval
 import pandas as pd
 from collections import defaultdict
 import json
 import copy
-import ir_datasets
+import plotly.express as px
 
 
-from constants import BEIR, IR_DATASETS, LOCAL_DATASETS
-
 
 @st.cache_data
 def load_local_corpus(corpus_file, columns_to_combine=["title", "text"]):
@@ -90,28 +84,6 @@ def load_local_qrels(qrels_file):
     return qid2did2label
 
 
-@st.cache_data
-def load_run(f_run):
-    run = pytrec_eval.parse_run(copy.deepcopy(f_run))
-    # convert bytes to strings for keys
-    new_run = defaultdict(dict)
-    for key, sub_dict in run.items():
-        new_run[key.decode("utf-8")] = {k.decode("utf-8"): v for k, v in sub_dict.items()}
-
-    run_pandas = pd.read_csv(f_run, header=None, index_col=None, sep="\t")
-    run_pandas.columns = ["qid", "generic", "doc_id", "rank", "score", "model"]
-    run_pandas.doc_id = run_pandas.doc_id.astype(str)
-    run_pandas.qid = run_pandas.qid.astype(str)
-    run_pandas["rank"] = run_pandas["rank"].astype(int)
-    run_pandas.score = run_pandas.score.astype(float)
-    all_groups = []
-    for qid, sub_df in run_pandas.groupby("qid"):
-        sub_df.sort_values(["score", "doc_id"], ascending=[False, False])
-        sub_df["rank"] = list(range(1, len(sub_df) + 1))
-        all_groups.append(sub_df)
-    run_pandas = pd.concat(all_groups)
-    return new_run, run_pandas
-
 
 @st.cache_data
 def load_jsonl(f):
@@ -137,46 +109,6 @@ def load_jsonl(f):
     return did2text, sub_did2text
 
 
-@st.cache_data(persist="disk")
-def get_beir(dataset: str):
-    url = "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/{}.zip".format(dataset)
-    out_dir = os.path.join(pathlib.Path(__file__).parent.absolute(), "datasets")
-    data_path = util.download_and_unzip(url, out_dir)
-    return GenericDataLoader(data_folder=data_path).load(split="test")
-
-
-@st.cache_data(persist="disk")
-def get_ir_datasets(dataset_name: str, input_fields_doc: str = None, input_fields_query: str = None):
-    dataset = ir_datasets.load(dataset_name)
-    queries = {}
-    for qid, query in dataset.queries_iter():
-        if input_fields_query is None: 
-            if type(query) == str:
-                queries[qid] = query
-            else:
-                # get all fields that exist in query
-                all_fields = {field: getattr(query, field) for field in query._fields}
-                # put all fields into a single string
-                queries[qid] = " ".join([str(v) for v in all_fields.values()])
-        else:
-            all_fields = {field: getattr(query, field) for field in input_fields_query}
-            queries[qid] = " ".join([str(v) for v in all_fields.values()])
-
-    corpus = {}
-    for doc in dataset.docs_iter():
-        if input_fields_doc is None:
-            if type(doc) == str:
-                corpus[doc.doc_id] = {"text": doc}
-            else: # get all fields that exist in query
-                all_fields = {field: getattr(doc, field) for field in doc._fields}
-                corpus[doc.doc_id] = {"text": " ".join([str(v) for v in all_fields.values()])}
-        else:
-            all_fields = {field: getattr(doc, field) for field in input_fields_doc}
-            corpus[doc.doc_id] = {"text": " ".join([str(v) for v in all_fields.values()])}
-
-    # return corpus, queries, qrels
-    return corpus, queries, dataset.qrels_dict()
-
 
 @st.cache_data(persist="disk")
 def get_dataset(dataset_name: str, input_fields_doc, input_fields_query):
@@ -188,15 +120,5 @@ def get_dataset(dataset_name: str, input_fields_doc, input_fields_query):
     if dataset_name == "":
         return {}, {}, {}
     
-    if dataset_name in BEIR:
-        return get_beir(dataset_name)
-    elif dataset_name in IR_DATASETS:
-        return get_ir_datasets(dataset_name, input_fields_doc, input_fields_query)
-    elif dataset_name in LOCAL_DATASETS:
-        base_path = f"local_datasets/{dataset_name}"
-        corpus_file = open(f"{base_path}/corpus.jsonl", "r")
-        queries_file = open(f"{base_path}/queries.jsonl", "r")
-        qrels_file = open(f"{base_path}/qrels/test.tsv", "r")
-        return load_local_corpus(corpus_file), load_local_queries(queries_file), load_local_qrels(qrels_file)
     else:
         raise NotImplementedError("Dataset not implemented")

ir_dataset_metadata.py

@@ -1,486 +0,0 @@
-
-IR_DATASETS = [
-    "antique/test",
-    "antique/test/non-offensive",
-    "antique/train",
-    "antique/train/split200-train",
-    "antique/train/split200-valid",
-    "aol-ia",
-    "aquaint/trec-robust-2005",
-    "argsme/1.0/touche-2020-task-1/uncorrected",
-    "argsme/2020-04-01/processed/touche-2022-task-1",
-    "argsme/2020-04-01/touche-2020-task-1",
-    "argsme/2020-04-01/touche-2020-task-1/uncorrected",
-    "argsme/2020-04-01/touche-2021-task-1",
-    "beir/arguana",
-    "beir/climate-fever",
-    "beir/cqadupstack/android",
-    "beir/cqadupstack/english",
-    "beir/cqadupstack/gaming",
-    "beir/cqadupstack/gis",
-    "beir/cqadupstack/mathematica",
-    "beir/cqadupstack/physics",
-    "beir/cqadupstack/programmers",
-    "beir/cqadupstack/stats",
-    "beir/cqadupstack/tex",
-    "beir/cqadupstack/unix",
-    "beir/cqadupstack/webmasters",
-    "beir/cqadupstack/wordpress",
-    "beir/dbpedia-entity/dev",
-    "beir/dbpedia-entity/test",
-    "beir/fever/dev",
-    "beir/fever/test",
-    "beir/fever/train",
-    "beir/fiqa/dev",
-    "beir/fiqa/test",
-    "beir/fiqa/train",
-    "beir/hotpotqa/dev",
-    "beir/hotpotqa/test",
-    "beir/hotpotqa/train",
-    "beir/msmarco/dev",
-    "beir/msmarco/test",
-    "beir/msmarco/train",
-    "beir/nfcorpus/dev",
-    "beir/nfcorpus/test",
-    "beir/nfcorpus/train",
-    "beir/nq",
-    "beir/quora/dev",
-    "beir/quora/test",
-    "beir/scidocs",
-    "beir/scifact/test",
-    "beir/scifact/train",
-    "beir/trec-covid",
-    "beir/webis-touche2020",
-    "beir/webis-touche2020/v2",
-    "car/v1.5/test200",
-    "car/v1.5/train/fold0",
-    "car/v1.5/train/fold1",
-    "car/v1.5/train/fold2",
-    "car/v1.5/train/fold3",
-    "car/v1.5/train/fold4",
-    "car/v1.5/trec-y1/auto",
-    "car/v1.5/trec-y1/manual",
-    "clinicaltrials/2017/trec-pm-2017",
-    "clinicaltrials/2017/trec-pm-2018",
-    "clinicaltrials/2019/trec-pm-2019",
-    "clinicaltrials/2021/trec-ct-2021",
-    "clueweb09/catb/trec-web-2009",
-    "clueweb09/catb/trec-web-2009/diversity",
-    "clueweb09/catb/trec-web-2010",
-    "clueweb09/catb/trec-web-2010/diversity",
-    "clueweb09/catb/trec-web-2011",
-    "clueweb09/catb/trec-web-2011/diversity",
-    "clueweb09/catb/trec-web-2012",
-    "clueweb09/catb/trec-web-2012/diversity",
-    "clueweb09/en/trec-web-2009",
-    "clueweb09/en/trec-web-2009/diversity",
-    "clueweb09/en/trec-web-2010",
-    "clueweb09/en/trec-web-2010/diversity",
-    "clueweb09/en/trec-web-2011",
-    "clueweb09/en/trec-web-2011/diversity",
-    "clueweb09/en/trec-web-2012",
-    "clueweb09/en/trec-web-2012/diversity",
-    "clueweb09/trec-mq-2009",
-    "clueweb12/b13/clef-ehealth",
-    "clueweb12/b13/clef-ehealth/cs",
-    "clueweb12/b13/clef-ehealth/de",
-    "clueweb12/b13/clef-ehealth/fr",
-    "clueweb12/b13/clef-ehealth/hu",
-    "clueweb12/b13/clef-ehealth/pl",
-    "clueweb12/b13/clef-ehealth/sv",
-    "clueweb12/b13/ntcir-www-1",
-    "clueweb12/b13/ntcir-www-2",
-    "clueweb12/b13/trec-misinfo-2019",
-    "clueweb12/touche-2020-task-2",
-    "clueweb12/touche-2021-task-2",
-    "clueweb12/touche-2022-task-2",
-    "clueweb12/touche-2022-task-2/expanded-doc-t5-query",
-    "clueweb12/trec-web-2013",
-    "clueweb12/trec-web-2013/diversity",
-    "clueweb12/trec-web-2014",
-    "clueweb12/trec-web-2014/diversity",
-    "codec",
-    "codec/economics",
-    "codec/history",
-    "codec/politics",
-    "codesearchnet/challenge",
-    "codesearchnet/test",
-    "codesearchnet/train",
-    "codesearchnet/valid",
-    "cord19/fulltext/trec-covid",
-    "cord19/trec-covid",
-    "cord19/trec-covid/round1",
-    "cord19/trec-covid/round2",
-    "cord19/trec-covid/round3",
-    "cord19/trec-covid/round4",
-    "cord19/trec-covid/round5",
-    "cranfield",
-    "disks45/nocr/trec-robust-2004",
-    "disks45/nocr/trec-robust-2004/fold1",
-    "disks45/nocr/trec-robust-2004/fold2",
-    "disks45/nocr/trec-robust-2004/fold3",
-    "disks45/nocr/trec-robust-2004/fold4",
-    "disks45/nocr/trec-robust-2004/fold5",
-    "disks45/nocr/trec7",
-    "disks45/nocr/trec8",
-    "dpr-w100/natural-questions/dev",
-    "dpr-w100/natural-questions/train",
-    "dpr-w100/trivia-qa/dev",
-    "dpr-w100/trivia-qa/train",
-    "gov/trec-web-2002",
-    "gov/trec-web-2002/named-page",
-    "gov/trec-web-2003",
-    "gov/trec-web-2003/named-page",
-    "gov/trec-web-2004",
-    "gov2/trec-mq-2007",
-    "gov2/trec-mq-2008",
-    "gov2/trec-tb-2004",
-    "gov2/trec-tb-2005",
-    "gov2/trec-tb-2005/efficiency",
-    "gov2/trec-tb-2005/named-page",
-    "gov2/trec-tb-2006",
-    "gov2/trec-tb-2006/efficiency",
-    "gov2/trec-tb-2006/efficiency/stream3",
-    "gov2/trec-tb-2006/named-page",
-    "hc4/fa/dev",
-    "hc4/fa/test",
-    "hc4/fa/train",
-    "hc4/ru/dev",
-    "hc4/ru/test",
-    "hc4/ru/train",
-    "hc4/zh/dev",
-    "hc4/zh/test",
-    "hc4/zh/train",
-    "highwire/trec-genomics-2006",
-    "highwire/trec-genomics-2007",
-    "istella22/test",
-    "istella22/test/fold1",
-    "istella22/test/fold2",
-    "istella22/test/fold3",
-    "istella22/test/fold4",
-    "istella22/test/fold5",
-    "kilt/codec",
-    "kilt/codec/economics",
-    "kilt/codec/history",
-    "kilt/codec/politics",
-    "lotte/lifestyle/dev/forum",
-    "lotte/lifestyle/dev/search",
-    "lotte/lifestyle/test/forum",
-    "lotte/lifestyle/test/search",
-    "lotte/pooled/dev/forum",
-    "lotte/pooled/dev/search",
-    "lotte/pooled/test/forum",
-    "lotte/pooled/test/search",
-    "lotte/recreation/dev/forum",
-    "lotte/recreation/dev/search",
-    "lotte/recreation/test/forum",
-    "lotte/recreation/test/search",
-    "lotte/science/dev/forum",
-    "lotte/science/dev/search",
-    "lotte/science/test/forum",
-    "lotte/science/test/search",
-    "lotte/technology/dev/forum",
-    "lotte/technology/dev/search",
-    "lotte/technology/test/forum",
-    "lotte/technology/test/search",
-    "lotte/writing/dev/forum",
-    "lotte/writing/dev/search",
-    "lotte/writing/test/forum",
-    "lotte/writing/test/search",
-    "medline/2004/trec-genomics-2004",
-    "medline/2004/trec-genomics-2005",
-    "medline/2017/trec-pm-2017",
-    "medline/2017/trec-pm-2018",
-    "mmarco/de/dev",
-    "mmarco/de/dev/small",
-    "mmarco/de/train",
-    "mmarco/es/dev",
-    "mmarco/es/dev/small",
-    "mmarco/es/train",
-    "mmarco/fr/dev",
-    "mmarco/fr/dev/small",
-    "mmarco/fr/train",
-    "mmarco/id/dev",
-    "mmarco/id/dev/small",
-    "mmarco/id/train",
-    "mmarco/it/dev",
-    "mmarco/it/dev/small",
-    "mmarco/it/train",
-    "mmarco/pt/dev",
-    "mmarco/pt/dev/small",
-    "mmarco/pt/dev/small/v1.1",
-    "mmarco/pt/dev/v1.1",
-    "mmarco/pt/train",
-    "mmarco/pt/train/v1.1",
-    "mmarco/ru/dev",
-    "mmarco/ru/dev/small",
-    "mmarco/ru/train",
-    "mmarco/v2/ar/dev",
-    "mmarco/v2/ar/dev/small",
-    "mmarco/v2/ar/train",
-    "mmarco/v2/de/dev",
-    "mmarco/v2/de/dev/small",
-    "mmarco/v2/de/train",
-    "mmarco/v2/dt/dev",
-    "mmarco/v2/dt/dev/small",
-    "mmarco/v2/dt/train",
-    "mmarco/v2/es/dev",
-    "mmarco/v2/es/dev/small",
-    "mmarco/v2/es/train",
-    "mmarco/v2/fr/dev",
-    "mmarco/v2/fr/dev/small",
-    "mmarco/v2/fr/train",
-    "mmarco/v2/hi/dev",
-    "mmarco/v2/hi/dev/small",
-    "mmarco/v2/hi/train",
-    "mmarco/v2/id/dev",
-    "mmarco/v2/id/dev/small",
-    "mmarco/v2/id/train",
-    "mmarco/v2/it/dev",
-    "mmarco/v2/it/dev/small",
-    "mmarco/v2/it/train",
-    "mmarco/v2/ja/dev",
-    "mmarco/v2/ja/dev/small",
-    "mmarco/v2/ja/train",
-    "mmarco/v2/pt/dev",
-    "mmarco/v2/pt/dev/small",
-    "mmarco/v2/pt/train",
-    "mmarco/v2/ru/dev",
-    "mmarco/v2/ru/dev/small",
-    "mmarco/v2/ru/train",
-    "mmarco/v2/vi/dev",
-    "mmarco/v2/vi/dev/small",
-    "mmarco/v2/vi/train",
-    "mmarco/v2/zh/dev",
-    "mmarco/v2/zh/dev/small",
-    "mmarco/v2/zh/train",
-    "mmarco/zh/dev",
-    "mmarco/zh/dev/small",
-    "mmarco/zh/dev/small/v1.1",
-    "mmarco/zh/dev/v1.1",
-    "mmarco/zh/train",
-    "mr-tydi/ar",
-    "mr-tydi/ar/dev",
-    "mr-tydi/ar/test",
-    "mr-tydi/ar/train",
-    "mr-tydi/bn",
-    "mr-tydi/bn/dev",
-    "mr-tydi/bn/test",
-    "mr-tydi/bn/train",
-    "mr-tydi/en",
-    "mr-tydi/en/dev",
-    "mr-tydi/en/test",
-    "mr-tydi/en/train",
-    "mr-tydi/fi",
-    "mr-tydi/fi/dev",
-    "mr-tydi/fi/test",
-    "mr-tydi/fi/train",
-    "mr-tydi/id",
-    "mr-tydi/id/dev",
-    "mr-tydi/id/test",
-    "mr-tydi/id/train",
-    "mr-tydi/ja",
-    "mr-tydi/ja/dev",
-    "mr-tydi/ja/test",
-    "mr-tydi/ja/train",
-    "mr-tydi/ko",
-    "mr-tydi/ko/dev",
-    "mr-tydi/ko/test",
-    "mr-tydi/ko/train",
-    "mr-tydi/ru",
-    "mr-tydi/ru/dev",
-    "mr-tydi/ru/test",
-    "mr-tydi/ru/train",
-    "mr-tydi/sw",
-    "mr-tydi/sw/dev",
-    "mr-tydi/sw/test",
-    "mr-tydi/sw/train",
-    "mr-tydi/te",
-    "mr-tydi/te/dev",
-    "mr-tydi/te/test",
-    "mr-tydi/te/train",
-    "mr-tydi/th",
-    "mr-tydi/th/dev",
-    "mr-tydi/th/test",
-    "mr-tydi/th/train",
-    "msmarco-document-v2/dev1",
-    "msmarco-document-v2/dev2",
-    "msmarco-document-v2/train",
-    "msmarco-document-v2/trec-dl-2019",
-    "msmarco-document-v2/trec-dl-2019/judged",
-    "msmarco-document-v2/trec-dl-2020",
-    "msmarco-document-v2/trec-dl-2020/judged",
-    "msmarco-document-v2/trec-dl-2021",
-    "msmarco-document-v2/trec-dl-2021/judged",
-    "msmarco-document-v2/trec-dl-2022",
-    "msmarco-document-v2/trec-dl-2022/judged",
-    "msmarco-document/dev",
-    "msmarco-document/orcas",
-    "msmarco-document/train",
-    "msmarco-document/trec-dl-2019",
-    "msmarco-document/trec-dl-2019/judged",
-    "msmarco-document/trec-dl-2020",
-    "msmarco-document/trec-dl-2020/judged",
-    "msmarco-document/trec-dl-hard",
-    "msmarco-document/trec-dl-hard/fold1",
-    "msmarco-document/trec-dl-hard/fold2",
-    "msmarco-document/trec-dl-hard/fold3",
-    "msmarco-document/trec-dl-hard/fold4",
-    "msmarco-document/trec-dl-hard/fold5",
-    "msmarco-passage-v2/dev1",
-    "msmarco-passage-v2/dev2",
-    "msmarco-passage-v2/train",
-    "msmarco-passage-v2/trec-dl-2021",
-    "msmarco-passage-v2/trec-dl-2021/judged",
-    "msmarco-passage-v2/trec-dl-2022",
-    "msmarco-passage-v2/trec-dl-2022/judged",
-    "msmarco-passage/dev",
-    "msmarco-passage/dev/2",
-    "msmarco-passage/dev/judged",
-    "msmarco-passage/dev/small",
-    "msmarco-passage/train",
-    "msmarco-passage/train/judged",
-    "msmarco-passage/train/medical",
-    "msmarco-passage/train/split200-train",
-    "msmarco-passage/train/split200-valid",
-    "msmarco-passage/train/triples-small",
-    "msmarco-passage/train/triples-v2",
-    "msmarco-passage/trec-dl-2019",
-    "msmarco-passage/trec-dl-2019/judged",
-    "msmarco-passage/trec-dl-2020",
-    "msmarco-passage/trec-dl-2020/judged",
-    "msmarco-passage/trec-dl-hard",
-    "msmarco-passage/trec-dl-hard/fold1",
-    "msmarco-passage/trec-dl-hard/fold2",
-    "msmarco-passage/trec-dl-hard/fold3",
-    "msmarco-passage/trec-dl-hard/fold4",
-    "msmarco-passage/trec-dl-hard/fold5",
-    "msmarco-qna/dev",
-    "msmarco-qna/train",
-    "natural-questions/dev",
-    "natural-questions/train",
-    "neuclir/1/fa/hc4-filtered",
-    "neuclir/1/ru/hc4-filtered",
-    "neuclir/1/zh/hc4-filtered",
-    "neumarco/fa/dev",
-    "neumarco/fa/dev/judged",
-    "neumarco/fa/dev/small",
-    "neumarco/fa/train",
-    "neumarco/fa/train/judged",
-    "neumarco/ru/dev",
-    "neumarco/ru/dev/judged",
-    "neumarco/ru/dev/small",
-    "neumarco/ru/train",
-    "neumarco/ru/train/judged",
-    "neumarco/zh/dev",
-    "neumarco/zh/dev/judged",
-    "neumarco/zh/dev/small",
-    "neumarco/zh/train",
-    "neumarco/zh/train/judged",
-    "nfcorpus/dev",
-    "nfcorpus/dev/nontopic",
-    "nfcorpus/dev/video",
-    "nfcorpus/test",
-    "nfcorpus/test/nontopic",
-    "nfcorpus/test/video",
-    "nfcorpus/train",
-    "nfcorpus/train/nontopic",
-    "nfcorpus/train/video",
-    "nyt/trec-core-2017",
-    "nyt/wksup",
-    "nyt/wksup/train",
-    "nyt/wksup/valid",
-    "pmc/v1/trec-cds-2014",
-    "pmc/v1/trec-cds-2015",
-    "pmc/v2/trec-cds-2016",
-    "sara",
-    "touche-image/2022-06-13/touche-2022-task-3",
-    "trec-arabic/ar2001",
-    "trec-arabic/ar2002",
-    "trec-cast/v0/train",
-    "trec-cast/v0/train/judged",
-    "trec-cast/v1/2019",
-    "trec-cast/v1/2019/judged",
-    "trec-cast/v1/2020",
-    "trec-cast/v1/2020/judged",
-    "trec-fair-2021/eval",
-    "trec-fair-2021/train",
-    "trec-fair/2021/eval",
-    "trec-fair/2021/train",
-    "trec-fair/2022/train",
-    "trec-mandarin/trec5",
-    "trec-mandarin/trec6",
-    "trec-robust04",
-    "trec-robust04/fold1",
-    "trec-robust04/fold2",
-    "trec-robust04/fold3",
-    "trec-robust04/fold4",
-    "trec-robust04/fold5",
-    "trec-spanish/trec3",
-    "trec-spanish/trec4",
-    "trec-tot/2023/dev",
-    "trec-tot/2023/train",
-    "tripclick/train",
-    "tripclick/train/head",
-    "tripclick/train/head/dctr",
-    "tripclick/train/hofstaetter-triples",
-    "tripclick/train/tail",
-    "tripclick/train/torso",
-    "tripclick/val",
-    "tripclick/val/head",
-    "tripclick/val/head/dctr",
-    "tripclick/val/tail",
-    "tripclick/val/torso",
-    "tweets2013-ia/trec-mb-2013",
-    "tweets2013-ia/trec-mb-2014",
-    "vaswani",
-    "wapo/v2/trec-core-2018",
-    "wapo/v2/trec-news-2018",
-    "wapo/v2/trec-news-2019",
-    "wikiclir/ar",
-    "wikiclir/ca",
-    "wikiclir/cs",
-    "wikiclir/de",
-    "wikiclir/en-simple",
-    "wikiclir/es",
-    "wikiclir/fi",
-    "wikiclir/fr",
-    "wikiclir/it",
-    "wikiclir/ja",
-    "wikiclir/ko",
-    "wikiclir/nl",
-    "wikiclir/nn",
-    "wikiclir/no",
-    "wikiclir/pl",
-    "wikiclir/pt",
-    "wikiclir/ro",
-    "wikiclir/ru",
-    "wikiclir/sv",
-    "wikiclir/sw",
-    "wikiclir/tl",
-    "wikiclir/tr",
-    "wikiclir/uk",
-    "wikiclir/vi",
-    "wikiclir/zh",
-    "wikir/en1k/test",
-    "wikir/en1k/training",
-    "wikir/en1k/validation",
-    "wikir/en59k/test",
-    "wikir/en59k/training",
-    "wikir/en59k/validation",
-    "wikir/en78k/test",
-    "wikir/en78k/training",
-    "wikir/en78k/validation",
-    "wikir/ens78k/test",
-    "wikir/ens78k/training",
-    "wikir/ens78k/validation",
-    "wikir/es13k/test",
-    "wikir/es13k/training",
-    "wikir/es13k/validation",
-    "wikir/fr14k/test",
-    "wikir/fr14k/training",
-    "wikir/fr14k/validation",
-    "wikir/it16k/test",
-    "wikir/it16k/training",
-    "wikir/it16k/validation"
-]

ir_dataset_names.json

@@ -1,485 +0,0 @@
-[
-    "antique/test",
-    "antique/test/non-offensive",
-    "antique/train",
-    "antique/train/split200-train",
-    "antique/train/split200-valid",
-    "aol-ia",
-    "aquaint/trec-robust-2005",
-    "argsme/1.0/touche-2020-task-1/uncorrected",
-    "argsme/2020-04-01/processed/touche-2022-task-1",
-    "argsme/2020-04-01/touche-2020-task-1",
-    "argsme/2020-04-01/touche-2020-task-1/uncorrected",
-    "argsme/2020-04-01/touche-2021-task-1",
-    "beir/arguana",
-    "beir/climate-fever",
-    "beir/cqadupstack/android",
-    "beir/cqadupstack/english",
-    "beir/cqadupstack/gaming",
-    "beir/cqadupstack/gis",
-    "beir/cqadupstack/mathematica",
-    "beir/cqadupstack/physics",
-    "beir/cqadupstack/programmers",
-    "beir/cqadupstack/stats",
-    "beir/cqadupstack/tex",
-    "beir/cqadupstack/unix",
-    "beir/cqadupstack/webmasters",
-    "beir/cqadupstack/wordpress",
-    "beir/dbpedia-entity/dev",
-    "beir/dbpedia-entity/test",
-    "beir/fever/dev",
-    "beir/fever/test",
-    "beir/fever/train",
-    "beir/fiqa/dev",
-    "beir/fiqa/test",
-    "beir/fiqa/train",
-    "beir/hotpotqa/dev",
-    "beir/hotpotqa/test",
-    "beir/hotpotqa/train",
-    "beir/msmarco/dev",
-    "beir/msmarco/test",
-    "beir/msmarco/train",
-    "beir/nfcorpus/dev",
-    "beir/nfcorpus/test",
-    "beir/nfcorpus/train",
-    "beir/nq",
-    "beir/quora/dev",
-    "beir/quora/test",
-    "beir/scidocs",
-    "beir/scifact/test",
-    "beir/scifact/train",
-    "beir/trec-covid",
-    "beir/webis-touche2020",
-    "beir/webis-touche2020/v2",
-    "car/v1.5/test200",
-    "car/v1.5/train/fold0",
-    "car/v1.5/train/fold1",
-    "car/v1.5/train/fold2",
-    "car/v1.5/train/fold3",
-    "car/v1.5/train/fold4",
-    "car/v1.5/trec-y1/auto",
-    "car/v1.5/trec-y1/manual",
-    "clinicaltrials/2017/trec-pm-2017",
-    "clinicaltrials/2017/trec-pm-2018",
-    "clinicaltrials/2019/trec-pm-2019",
-    "clinicaltrials/2021/trec-ct-2021",
-    "clueweb09/catb/trec-web-2009",
-    "clueweb09/catb/trec-web-2009/diversity",
-    "clueweb09/catb/trec-web-2010",
-    "clueweb09/catb/trec-web-2010/diversity",
-    "clueweb09/catb/trec-web-2011",
-    "clueweb09/catb/trec-web-2011/diversity",
-    "clueweb09/catb/trec-web-2012",
-    "clueweb09/catb/trec-web-2012/diversity",
-    "clueweb09/en/trec-web-2009",
-    "clueweb09/en/trec-web-2009/diversity",
-    "clueweb09/en/trec-web-2010",
-    "clueweb09/en/trec-web-2010/diversity",
-    "clueweb09/en/trec-web-2011",
-    "clueweb09/en/trec-web-2011/diversity",
-    "clueweb09/en/trec-web-2012",
-    "clueweb09/en/trec-web-2012/diversity",
-    "clueweb09/trec-mq-2009",
-    "clueweb12/b13/clef-ehealth",
-    "clueweb12/b13/clef-ehealth/cs",
-    "clueweb12/b13/clef-ehealth/de",
-    "clueweb12/b13/clef-ehealth/fr",
-    "clueweb12/b13/clef-ehealth/hu",
-    "clueweb12/b13/clef-ehealth/pl",
-    "clueweb12/b13/clef-ehealth/sv",
-    "clueweb12/b13/ntcir-www-1",
-    "clueweb12/b13/ntcir-www-2",
-    "clueweb12/b13/trec-misinfo-2019",
-    "clueweb12/touche-2020-task-2",
-    "clueweb12/touche-2021-task-2",
-    "clueweb12/touche-2022-task-2",
-    "clueweb12/touche-2022-task-2/expanded-doc-t5-query",
-    "clueweb12/trec-web-2013",
-    "clueweb12/trec-web-2013/diversity",
-    "clueweb12/trec-web-2014",
-    "clueweb12/trec-web-2014/diversity",
-    "codec",
-    "codec/economics",
-    "codec/history",
-    "codec/politics",
-    "codesearchnet/challenge",
-    "codesearchnet/test",
-    "codesearchnet/train",
-    "codesearchnet/valid",
-    "cord19/fulltext/trec-covid",
-    "cord19/trec-covid",
-    "cord19/trec-covid/round1",
-    "cord19/trec-covid/round2",
-    "cord19/trec-covid/round3",
-    "cord19/trec-covid/round4",
-    "cord19/trec-covid/round5",
-    "cranfield",
-    "disks45/nocr/trec-robust-2004",
-    "disks45/nocr/trec-robust-2004/fold1",
-    "disks45/nocr/trec-robust-2004/fold2",
-    "disks45/nocr/trec-robust-2004/fold3",
-    "disks45/nocr/trec-robust-2004/fold4",
-    "disks45/nocr/trec-robust-2004/fold5",
-    "disks45/nocr/trec7",
-    "disks45/nocr/trec8",
-    "dpr-w100/natural-questions/dev",
-    "dpr-w100/natural-questions/train",
-    "dpr-w100/trivia-qa/dev",
-    "dpr-w100/trivia-qa/train",
-    "gov/trec-web-2002",
-    "gov/trec-web-2002/named-page",
-    "gov/trec-web-2003",
-    "gov/trec-web-2003/named-page",
-    "gov/trec-web-2004",
-    "gov2/trec-mq-2007",
-    "gov2/trec-mq-2008",
-    "gov2/trec-tb-2004",
-    "gov2/trec-tb-2005",
-    "gov2/trec-tb-2005/efficiency",
-    "gov2/trec-tb-2005/named-page",
-    "gov2/trec-tb-2006",
-    "gov2/trec-tb-2006/efficiency",
-    "gov2/trec-tb-2006/efficiency/stream3",
-    "gov2/trec-tb-2006/named-page",
-    "hc4/fa/dev",
-    "hc4/fa/test",
-    "hc4/fa/train",
-    "hc4/ru/dev",
-    "hc4/ru/test",
-    "hc4/ru/train",
-    "hc4/zh/dev",
-    "hc4/zh/test",
-    "hc4/zh/train",
-    "highwire/trec-genomics-2006",
-    "highwire/trec-genomics-2007",
-    "istella22/test",
-    "istella22/test/fold1",
-    "istella22/test/fold2",
-    "istella22/test/fold3",
-    "istella22/test/fold4",
-    "istella22/test/fold5",
-    "kilt/codec",
-    "kilt/codec/economics",
-    "kilt/codec/history",
-    "kilt/codec/politics",
-    "lotte/lifestyle/dev/forum",
-    "lotte/lifestyle/dev/search",
-    "lotte/lifestyle/test/forum",
-    "lotte/lifestyle/test/search",
-    "lotte/pooled/dev/forum",
-    "lotte/pooled/dev/search",
-    "lotte/pooled/test/forum",
-    "lotte/pooled/test/search",
-    "lotte/recreation/dev/forum",
-    "lotte/recreation/dev/search",
-    "lotte/recreation/test/forum",
-    "lotte/recreation/test/search",
-    "lotte/science/dev/forum",
-    "lotte/science/dev/search",
-    "lotte/science/test/forum",
-    "lotte/science/test/search",
-    "lotte/technology/dev/forum",
-    "lotte/technology/dev/search",
-    "lotte/technology/test/forum",
-    "lotte/technology/test/search",
-    "lotte/writing/dev/forum",
-    "lotte/writing/dev/search",
-    "lotte/writing/test/forum",
-    "lotte/writing/test/search",
-    "medline/2004/trec-genomics-2004",
-    "medline/2004/trec-genomics-2005",
-    "medline/2017/trec-pm-2017",
-    "medline/2017/trec-pm-2018",
-    "mmarco/de/dev",
-    "mmarco/de/dev/small",
-    "mmarco/de/train",
-    "mmarco/es/dev",
-    "mmarco/es/dev/small",
-    "mmarco/es/train",
-    "mmarco/fr/dev",
-    "mmarco/fr/dev/small",
-    "mmarco/fr/train",
-    "mmarco/id/dev",
-    "mmarco/id/dev/small",
-    "mmarco/id/train",
-    "mmarco/it/dev",
-    "mmarco/it/dev/small",
-    "mmarco/it/train",
-    "mmarco/pt/dev",
-    "mmarco/pt/dev/small",
-    "mmarco/pt/dev/small/v1.1",
-    "mmarco/pt/dev/v1.1",
-    "mmarco/pt/train",
-    "mmarco/pt/train/v1.1",
-    "mmarco/ru/dev",
-    "mmarco/ru/dev/small",
-    "mmarco/ru/train",
-    "mmarco/v2/ar/dev",
-    "mmarco/v2/ar/dev/small",
-    "mmarco/v2/ar/train",
-    "mmarco/v2/de/dev",
-    "mmarco/v2/de/dev/small",
-    "mmarco/v2/de/train",
-    "mmarco/v2/dt/dev",
-    "mmarco/v2/dt/dev/small",
-    "mmarco/v2/dt/train",
-    "mmarco/v2/es/dev",
-    "mmarco/v2/es/dev/small",
-    "mmarco/v2/es/train",
-    "mmarco/v2/fr/dev",
-    "mmarco/v2/fr/dev/small",
-    "mmarco/v2/fr/train",
-    "mmarco/v2/hi/dev",
-    "mmarco/v2/hi/dev/small",
-    "mmarco/v2/hi/train",
-    "mmarco/v2/id/dev",
-    "mmarco/v2/id/dev/small",
-    "mmarco/v2/id/train",
-    "mmarco/v2/it/dev",
-    "mmarco/v2/it/dev/small",
-    "mmarco/v2/it/train",
-    "mmarco/v2/ja/dev",
-    "mmarco/v2/ja/dev/small",
-    "mmarco/v2/ja/train",
-    "mmarco/v2/pt/dev",
-    "mmarco/v2/pt/dev/small",
-    "mmarco/v2/pt/train",
-    "mmarco/v2/ru/dev",
-    "mmarco/v2/ru/dev/small",
-    "mmarco/v2/ru/train",
-    "mmarco/v2/vi/dev",
-    "mmarco/v2/vi/dev/small",
-    "mmarco/v2/vi/train",
-    "mmarco/v2/zh/dev",
-    "mmarco/v2/zh/dev/small",
-    "mmarco/v2/zh/train",
-    "mmarco/zh/dev",
-    "mmarco/zh/dev/small",
-    "mmarco/zh/dev/small/v1.1",
-    "mmarco/zh/dev/v1.1",
-    "mmarco/zh/train",
-    "mr-tydi/ar",
-    "mr-tydi/ar/dev",
-    "mr-tydi/ar/test",
-    "mr-tydi/ar/train",
-    "mr-tydi/bn",
-    "mr-tydi/bn/dev",
-    "mr-tydi/bn/test",
-    "mr-tydi/bn/train",
-    "mr-tydi/en",
-    "mr-tydi/en/dev",
-    "mr-tydi/en/test",
-    "mr-tydi/en/train",
-    "mr-tydi/fi",
-    "mr-tydi/fi/dev",
-    "mr-tydi/fi/test",
-    "mr-tydi/fi/train",
-    "mr-tydi/id",
-    "mr-tydi/id/dev",
-    "mr-tydi/id/test",
-    "mr-tydi/id/train",
-    "mr-tydi/ja",
-    "mr-tydi/ja/dev",
-    "mr-tydi/ja/test",
-    "mr-tydi/ja/train",
-    "mr-tydi/ko",
-    "mr-tydi/ko/dev",
-    "mr-tydi/ko/test",
-    "mr-tydi/ko/train",
-    "mr-tydi/ru",
-    "mr-tydi/ru/dev",
-    "mr-tydi/ru/test",
-    "mr-tydi/ru/train",
-    "mr-tydi/sw",
-    "mr-tydi/sw/dev",
-    "mr-tydi/sw/test",
-    "mr-tydi/sw/train",
-    "mr-tydi/te",
-    "mr-tydi/te/dev",
-    "mr-tydi/te/test",
-    "mr-tydi/te/train",
-    "mr-tydi/th",
-    "mr-tydi/th/dev",
-    "mr-tydi/th/test",
-    "mr-tydi/th/train",
-    "msmarco-document-v2/dev1",
-    "msmarco-document-v2/dev2",
-    "msmarco-document-v2/train",
-    "msmarco-document-v2/trec-dl-2019",
-    "msmarco-document-v2/trec-dl-2019/judged",
-    "msmarco-document-v2/trec-dl-2020",
-    "msmarco-document-v2/trec-dl-2020/judged",
-    "msmarco-document-v2/trec-dl-2021",
-    "msmarco-document-v2/trec-dl-2021/judged",
-    "msmarco-document-v2/trec-dl-2022",
-    "msmarco-document-v2/trec-dl-2022/judged",
-    "msmarco-document/dev",
-    "msmarco-document/orcas",
-    "msmarco-document/train",
-    "msmarco-document/trec-dl-2019",
-    "msmarco-document/trec-dl-2019/judged",
-    "msmarco-document/trec-dl-2020",
-    "msmarco-document/trec-dl-2020/judged",
-    "msmarco-document/trec-dl-hard",
-    "msmarco-document/trec-dl-hard/fold1",
-    "msmarco-document/trec-dl-hard/fold2",
-    "msmarco-document/trec-dl-hard/fold3",
-    "msmarco-document/trec-dl-hard/fold4",
-    "msmarco-document/trec-dl-hard/fold5",
-    "msmarco-passage-v2/dev1",
-    "msmarco-passage-v2/dev2",
-    "msmarco-passage-v2/train",
-    "msmarco-passage-v2/trec-dl-2021",
-    "msmarco-passage-v2/trec-dl-2021/judged",
-    "msmarco-passage-v2/trec-dl-2022",
-    "msmarco-passage-v2/trec-dl-2022/judged",
-    "msmarco-passage/dev",
-    "msmarco-passage/dev/2",
-    "msmarco-passage/dev/judged",
-    "msmarco-passage/dev/small",
-    "msmarco-passage/train",
-    "msmarco-passage/train/judged",
-    "msmarco-passage/train/medical",
-    "msmarco-passage/train/split200-train",
-    "msmarco-passage/train/split200-valid",
-    "msmarco-passage/train/triples-small",
-    "msmarco-passage/train/triples-v2",
-    "msmarco-passage/trec-dl-2019",
-    "msmarco-passage/trec-dl-2019/judged",
-    "msmarco-passage/trec-dl-2020",
-    "msmarco-passage/trec-dl-2020/judged",
-    "msmarco-passage/trec-dl-hard",
-    "msmarco-passage/trec-dl-hard/fold1",
-    "msmarco-passage/trec-dl-hard/fold2",
-    "msmarco-passage/trec-dl-hard/fold3",
-    "msmarco-passage/trec-dl-hard/fold4",
-    "msmarco-passage/trec-dl-hard/fold5",
-    "msmarco-qna/dev",
-    "msmarco-qna/train",
-    "natural-questions/dev",
-    "natural-questions/train",
-    "neuclir/1/fa/hc4-filtered",
-    "neuclir/1/ru/hc4-filtered",
-    "neuclir/1/zh/hc4-filtered",
-    "neumarco/fa/dev",
-    "neumarco/fa/dev/judged",
-    "neumarco/fa/dev/small",
-    "neumarco/fa/train",
-    "neumarco/fa/train/judged",
-    "neumarco/ru/dev",
-    "neumarco/ru/dev/judged",
-    "neumarco/ru/dev/small",
-    "neumarco/ru/train",
-    "neumarco/ru/train/judged",
-    "neumarco/zh/dev",
-    "neumarco/zh/dev/judged",
-    "neumarco/zh/dev/small",
-    "neumarco/zh/train",
-    "neumarco/zh/train/judged",
-    "nfcorpus/dev",
-    "nfcorpus/dev/nontopic",
-    "nfcorpus/dev/video",
-    "nfcorpus/test",
-    "nfcorpus/test/nontopic",
-    "nfcorpus/test/video",
-    "nfcorpus/train",
-    "nfcorpus/train/nontopic",
-    "nfcorpus/train/video",
-    "nyt/trec-core-2017",
-    "nyt/wksup",
-    "nyt/wksup/train",
-    "nyt/wksup/valid",
-    "pmc/v1/trec-cds-2014",
-    "pmc/v1/trec-cds-2015",
-    "pmc/v2/trec-cds-2016",
-    "sara",
-    "touche-image/2022-06-13/touche-2022-task-3",
-    "trec-arabic/ar2001",
-    "trec-arabic/ar2002",
-    "trec-cast/v0/train",
-    "trec-cast/v0/train/judged",
-    "trec-cast/v1/2019",
-    "trec-cast/v1/2019/judged",
-    "trec-cast/v1/2020",
-    "trec-cast/v1/2020/judged",
-    "trec-fair-2021/eval",
-    "trec-fair-2021/train",
-    "trec-fair/2021/eval",
-    "trec-fair/2021/train",
-    "trec-fair/2022/train",
-    "trec-mandarin/trec5",
-    "trec-mandarin/trec6",
-    "trec-robust04",
-    "trec-robust04/fold1",
-    "trec-robust04/fold2",
-    "trec-robust04/fold3",
-    "trec-robust04/fold4",
-    "trec-robust04/fold5",
-    "trec-spanish/trec3",
-    "trec-spanish/trec4",
-    "trec-tot/2023/dev",
-    "trec-tot/2023/train",
-    "tripclick/train",
-    "tripclick/train/head",
-    "tripclick/train/head/dctr",
-    "tripclick/train/hofstaetter-triples",
-    "tripclick/train/tail",
-    "tripclick/train/torso",
-    "tripclick/val",
-    "tripclick/val/head",
-    "tripclick/val/head/dctr",
-    "tripclick/val/tail",
-    "tripclick/val/torso",
-    "tweets2013-ia/trec-mb-2013",
-    "tweets2013-ia/trec-mb-2014",
-    "vaswani",
-    "wapo/v2/trec-core-2018",
-    "wapo/v2/trec-news-2018",
-    "wapo/v2/trec-news-2019",
-    "wikiclir/ar",
-    "wikiclir/ca",
-    "wikiclir/cs",
-    "wikiclir/de",
-    "wikiclir/en-simple",
-    "wikiclir/es",
-    "wikiclir/fi",
-    "wikiclir/fr",
-    "wikiclir/it",
-    "wikiclir/ja",
-    "wikiclir/ko",
-    "wikiclir/nl",
-    "wikiclir/nn",
-    "wikiclir/no",
-    "wikiclir/pl",
-    "wikiclir/pt",
-    "wikiclir/ro",
-    "wikiclir/ru",
-    "wikiclir/sv",
-    "wikiclir/sw",
-    "wikiclir/tl",
-    "wikiclir/tr",
-    "wikiclir/uk",
-    "wikiclir/vi",
-    "wikiclir/zh",
-    "wikir/en1k/test",
-    "wikir/en1k/training",
-    "wikir/en1k/validation",
-    "wikir/en59k/test",
-    "wikir/en59k/training",
-    "wikir/en59k/validation",
-    "wikir/en78k/test",
-    "wikir/en78k/training",
-    "wikir/en78k/validation",
-    "wikir/ens78k/test",
-    "wikir/ens78k/training",
-    "wikir/ens78k/validation",
-    "wikir/es13k/test",
-    "wikir/es13k/training",
-    "wikir/es13k/validation",
-    "wikir/fr14k/test",
-    "wikir/fr14k/training",
-    "wikir/fr14k/validation",
-    "wikir/it16k/test",
-    "wikir/it16k/training",
-    "wikir/it16k/validation"
-]

local_datasets/codesearch_py/corpus.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:87ef61791e9aa9a9833e59e81756d41beaca8e4cd3efad2bb8940e5876f69008
-size 384365716

local_datasets/codesearch_py/qrels/test.tsv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d172966a5e2dcc39491d446ca75ed730f7309d09701c131add14eb62b45c2114
-size 79309

local_datasets/codesearch_py/qrels/test.tsv.tmp

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ef54b582e47e48fdd094a3da00644bcf4af684b709be3f4f72f4de23c783ea50
-size 79283

local_datasets/codesearch_py/qrels/test.tsv.tmp.2

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:efda84b6d2b19a4bbd33ecd89616c88b63f4d585f7cb5ea10cc12372592306a3
-size 81283

local_datasets/codesearch_py/qrels/test.tsv.tmp.2.filtered

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ea798baa1ab23010a7769e60ba06e388d2b421cc2a9987b13900743df122a7c2
-size 24193

local_datasets/codesearch_py/queries.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:41e3f41fad388f4f612630bdb8ccb23b319b24a0b859db226a381b6f68b1771c
-size 199567

local_datasets/gooaq_technical/corpus.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:31282e5019461a6cd9d88a9e47fe6743d6962b3aeb81f5f5f78fa72eb52ff46b
-size 1399723

local_datasets/gooaq_technical/qrels/test.tsv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0b56de4bfec42225780cda2fc28fd7e0ee433f313208ab210de5bcf6281757ee
-size 49675

local_datasets/gooaq_technical/qrels/test.tsv.tmp

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:49982dbf8d1d182a75935718cb183b91d29e3ad4db1892723371c7d762955cbc
-size 49649

local_datasets/gooaq_technical/qrels/test.tsv.tmp.2

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5f0c2a26846e0456ddd24cd6d315ae516af28504e6b2961d00e0da0ff821f648
-size 51649

local_datasets/gooaq_technical/qrels/test.tsv.tmp.2.filtered

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e681ddae1619d30ce425fdb01ca4ceb10f493b079369ac0e555b1338cd3914e1
-size 15158

local_datasets/gooaq_technical/queries.jsonl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:25df519a5e39f0c91f6f7c5bdb74601cbfffbadd3cd1a998a8a5a48740b885be
-size 110860

requirements.txt

@@ -1,11 +1,5 @@
-beir==1.0.1
 pandas==2.0.3
-pytrec_eval==0.5
 streamlit==1.24.1
-ir_datasets==0.5.5
-pyserini==0.21.0
-torch==2.0.1
 plotly==5.15.0
-captum==0.6.0
 protobuf==3.20.0
 beautifulsoup4==4.12.2

scripts/collect_ir_dataset_names.py

@@ -1,26 +0,0 @@
-import requests
-from bs4 import BeautifulSoup
-import re
-import json
-import os
-import pathlib
-import shutil
-
-
-
-def get_ir_dataset_names():
-    url = "https://raw.githubusercontent.com/allenai/ir_datasets/master/ir_datasets/etc/metadata.json"
-    # read in the json
-    with requests.get(url) as r:
-        data = json.loads(r.text)
-    names = []
-    for dataset in data:
-        if "docs" in data[dataset] and "queries" in data[dataset] and "qrels" in data[dataset]:
-            names.append(dataset)
-    return names
-
-
-if __name__ == "__main__":
-    names = get_ir_dataset_names()
-    with open("ir_dataset_names.json", "w") as fout:
-        json.dump(names, fout, indent=4)

test.tst

@@ -1,55 +0,0 @@
-"base",
-"antique",
-"aol_ia",
-"aquaint",
-"argsme",
-"beir",
-"c4
-"car",
-"clinicaltrials",
-"clirmatrix",
-"clueweb09",
-"clueweb12",
-"codec",
-"cord19",
-"cranfield",
-"disks45",
-"dpr_w100",
-"codesearchnet",
-"gov",
-"gov2",
-"highwire",
-"istella22",
-"kilt",
-"lotte",
-"medline",
-"mmarco",
-"mr_tydi",
-"msmarco_document",
-"msmarco_document_v2",
-"msmarco_passage",
-"msmarco_passage_v2",
-"msmarco_qna",
-"neumarco",
-"nfcorpus",
-"natural_questions",
-"nyt",
-"pmc",
-"touche_image",
-"touche",
-"trec_arabic",
-"trec_mandarin",
-"trec_spanish",
-"trec_robust04",
-"trec_tot",
-"tripclick",
-"tweets2013_ia",
-"vaswani",
-"wapo",
-"wikiclir",
-"wikir",
-"trec_fair",
-"trec_cast",
-"hc4",
-"neuclir",
-"sara",