|
|
""" |
|
|
QuAC: Question Answering in Context |
|
|
https://arxiv.org/abs/1808.07036 |
|
|
|
|
|
Question Answering in Context (QuAC) is a dataset for modeling, understanding, and |
|
|
participating in information seeking dialog. Data instances consist of an interactive |
|
|
dialog between two crowd workers: (1) a student who poses a sequence of freeform |
|
|
questions to learn as much as possible about a hidden Wikipedia text, and (2) |
|
|
a teacher who answers the questions by providing short excerpts (spans) from the text. |
|
|
|
|
|
Homepage: https://quac.ai/ |
|
|
""" |
|
|
import inspect |
|
|
import lm_eval.datasets.quac.quac |
|
|
from lm_eval.base import Task |
|
|
|
|
|
|
|
|
_CITATION = """ |
|
|
@article{choi2018quac, |
|
|
title={Quac: Question answering in context}, |
|
|
author={Choi, Eunsol and He, He and Iyyer, Mohit and Yatskar, Mark and Yih, Wen-tau and Choi, Yejin and Liang, Percy and Zettlemoyer, Luke}, |
|
|
journal={arXiv preprint arXiv:1808.07036}, |
|
|
year={2018} |
|
|
} |
|
|
""" |
|
|
|
|
|
|
|
|
class QuAC(Task): |
|
|
VERSION = 0 |
|
|
DATASET_PATH = inspect.getfile(lm_eval.datasets.quac.quac) |
|
|
DATASET_NAME = None |
|
|
|
|
|
def has_training_docs(self): |
|
|
return True |
|
|
|
|
|
def has_validation_docs(self): |
|
|
return True |
|
|
|
|
|
def has_test_docs(self): |
|
|
return False |
|
|
|
|
|
def training_docs(self): |
|
|
if self._training_docs is None: |
|
|
self._training_docs = list(map(self._process_doc, self.dataset["train"])) |
|
|
return self._training_docs |
|
|
|
|
|
def validation_docs(self): |
|
|
return map(self._process_doc, self.dataset["validation"]) |
|
|
|
|
|
def test_docs(self): |
|
|
raise NotImplementedError("QuAC has no test docs.") |
|
|
|
|
|
def _process_doc(self, doc): |
|
|
doc["title"] = doc["title"] + " - " + doc["section_title"] |
|
|
return doc |
|
|
|
|
|
def doc_to_text(self, doc): |
|
|
return ( |
|
|
"TITLE: " |
|
|
+ doc["title"] |
|
|
+ "\n" |
|
|
+ "PARAGRAPH: " |
|
|
+ doc["paragraph"] |
|
|
+ "\n\n" |
|
|
+ "Q: " |
|
|
+ doc["question"] |
|
|
+ "\n\n" |
|
|
+ "A: " |
|
|
) |
|
|
|
|
|
def should_decontaminate(self): |
|
|
return True |
|
|
|
|
|
def doc_to_decontamination_query(self, doc): |
|
|
return doc["paragraph"] |
|
|
|
|
|
def doc_to_target(self, doc): |
|
|
return doc["answer"] |
|
|
|
|
|
def construct_requests(self, doc, ctx): |
|
|
"""Uses RequestFactory to construct Requests and returns an iterable of |
|
|
Requests which will be sent to the LM. |
|
|
|
|
|
:param doc: |
|
|
The document as returned from training_docs, validation_docs, or test_docs. |
|
|
:param ctx: str |
|
|
The context string, generated by fewshot_context. This includes the natural |
|
|
language description, as well as the few shot examples, and the question |
|
|
part of the document for `doc`. |
|
|
""" |
|
|
|
|
|
raise NotImplementedError("Evaluation not implemented") |
|
|
|
|
|
def process_results(self, doc, results): |
|
|
"""Take a single document and the LM results and evaluates, returning a |
|
|
dict where keys are the names of submetrics and values are the values of |
|
|
the metric for that one document |
|
|
|
|
|
:param doc: |
|
|
The document as returned from training_docs, validation_docs, or test_docs. |
|
|
:param results: |
|
|
The results of the requests created in construct_requests. |
|
|
""" |
|
|
|
|
|
raise NotImplementedError("Evaluation not implemented") |
|
|
|
|
|
def aggregation(self): |
|
|
""" |
|
|
:returns: {str: [float] -> float} |
|
|
A dictionary where keys are the names of submetrics and values are |
|
|
functions that aggregate a list of metrics |
|
|
""" |
|
|
|
|
|
raise NotImplementedError("Evaluation not implemented") |
|
|
|
|
|
def higher_is_better(self): |
|
|
""" |
|
|
:returns: {str: bool} |
|
|
A dictionary where keys are the names of submetrics and values are |
|
|
whether a higher value of the submetric is better |
|
|
""" |
|
|
|
|
|
raise NotImplementedError("Evaluation not implemented") |
|
|
|
