Spaces:

aliabd
/

SummerTime

Build error

SummerTime / dataset /dataset_loaders.py

aliabd

full demo working with old graido

7e3e85d almost 2 years ago

No virus

16.6 kB

	from os import path
	from tqdm import tqdm
	from typing import List, Generator, Optional, Union

	from datasets import Dataset

	from dataset.st_dataset import SummInstance, SummDataset


	# Set directory to load non_huggingface dataset scripts
	FILE_DIRECTORY_PATH = path.dirname(path.realpath(__file__))
	BASE_NONHUGGINGFACE_DATASETS_PATH = path.join(
	FILE_DIRECTORY_PATH, "non_huggingface_datasets_builders"
	)


	# Huggingface Datasets


	class CnndmDataset(SummDataset):
	"""
	The CNN/DM dataset
	"""

	dataset_name = "CNN/DailyMail"

	is_query_based = False
	is_dialogue_based = False
	is_multi_document = False

	huggingface_dataset = True
	huggingface_page = "https://huggingface.co/datasets/cnn_dailymail"

	def __init__(self):
	super().__init__(
	dataset_args=(
	"cnn_dailymail",
	"3.0.0",
	)
	)

	def _process_data(self, data: Dataset) -> Generator[SummInstance, None, None]:
	"""
	Overrides the SummDataset '_process_data()' method
	This method processes the data contained in the dataset
	and puts each data instance into a SummInstance object
	:param dataset: a train/validation/test dataset
	:rtype: a generator yielding SummInstance objects
	"""
	for instance in tqdm(data):
	article: str = instance["article"]
	highlights: str = instance["highlights"]
	summ_instance = SummInstance(source=article, summary=highlights)

	yield summ_instance


	class MultinewsDataset(SummDataset):
	"""
	The Multi News dataset
	"""

	dataset_name = "Multinews"

	is_query_based = False
	is_dialogue_based = False
	is_multi_document = True

	huggingface_dataset = True
	huggingface_page = "https://huggingface.co/datasets/multi_news"

	def __init__(self):
	super().__init__(dataset_args=("multi_news",))

	def _process_data(self, data: Dataset) -> Generator[SummInstance, None, None]:
	"""
	Overrides the SummDataset '_process_data()' method
	This method processes the data contained in the dataset
	and puts each data instance into a SummInstance object
	:param dataset: a train/validation/test dataset
	:rtype: a generator yielding SummInstance objects
	"""
	for instance in tqdm(data):
	document: list = [
	doc for doc in instance["document"].split("\|\|\|\|\|") if doc
	] # removes the empty string generated
	# since each doc ends with the delimiting token '\|\|\|\|\|'
	# the final doc creates an empty string
	summary: str = instance["summary"]
	summ_instance = SummInstance(source=document, summary=summary)

	yield summ_instance


	class SamsumDataset(SummDataset):
	"""
	The SAMsum Dataset
	"""

	dataset_name = "Samsum"

	is_query_based = False
	is_dialogue_based = True
	is_multi_document = False

	huggingface_dataset = True
	huggingface_page = "https://huggingface.co/datasets/samsum"

	def __init__(self):
	super().__init__(dataset_args=("samsum",))

	def _process_data(self, data: Dataset) -> Generator[SummInstance, None, None]:
	"""
	Overrides the SummDataset '_process_data()' method
	This method processes the data contained in the dataset
	and puts each data instance into a SummInstance object
	:param dataset: a train/validation/test dataset
	:rtype: a generator yielding SummInstance objects
	"""
	for instance in tqdm(data):
	dialogue: List = instance["dialogue"].split(
	"\r\n"
	) # split each dialogue into a list of strings such as
	# ["speaker1 : utter..", "speaker2 : utter..."]
	summary: str = instance["summary"]
	summ_instance = SummInstance(source=dialogue, summary=summary)

	yield summ_instance


	class XsumDataset(SummDataset):
	"""
	The Xsum Dataset
	"""

	dataset_name = "Xsum"

	huggingface_dataset = True
	huggingface_page = "https://huggingface.co/datasets/xsum"

	is_query_based = False
	is_dialogue_based = False
	is_multi_document = False

	def __init__(self):
	super().__init__(dataset_args=("xsum",))

	def _process_data(self, data: Dataset) -> Generator[SummInstance, None, None]:
	"""
	Overrides the SummDataset '_process_data()' method
	This method processes the data contained in the dataset
	and puts each data instance into a SummInstance object
	:param dataset: a train/validation/test dataset
	:rtype: a generator yielding SummInstance objects
	"""
	for instance in tqdm(data):
	document: List = instance["document"]
	summary: str = instance["summary"]
	summ_instance = SummInstance(source=document, summary=summary)

	yield summ_instance


	class PubmedqaDataset(SummDataset):
	"""
	The Pubmed QA dataset
	"""

	dataset_name = "Pubmedqa"

	is_query_based = True
	is_dialogue_based = False
	is_multi_document = False

	huggingface_dataset = True
	huggingface_page = "https://huggingface.co/datasets/pubmed_qa"

	def __init__(self, seed=None):
	super().__init__(
	dataset_args=(
	"pubmed_qa",
	"pqa_artificial",
	)
	)

	def _process_data(self, data: Dataset) -> Generator[SummInstance, None, None]:
	"""
	Overrides the SummDataset '_process_data()' method
	This method processes the data contained in the dataset
	and puts each data instance into a SummInstance object
	:param dataset: a train/validation/test dataset
	:rtype: a generator yielding SummInstance objects
	"""
	for instance in tqdm(data):
	context: str = " ".join(instance["context"]["contexts"])
	answer: str = instance["long_answer"]
	query: str = instance["question"]
	summ_instance = SummInstance(source=context, summary=answer, query=query)

	yield summ_instance


	class MlsumDataset(SummDataset):
	"""
	The MLsum Dataset - A multi-lingual dataset featuring 5 languages
	Includes 1.5 million news articles and their corresponding summaries

	"de" - German
	"es" - Spanish
	"fr" - French
	"ru" - Russian
	"tu" - Turkish
	"""

	dataset_name = "MlSum"

	is_query_based = False
	is_dialogue_based = False
	is_multi_document = False

	huggingface_dataset = True
	huggingface_page = "https://huggingface.co/datasets/mlsum"
	supported_languages = ["de", "es", "fr", "ru", "tu"]

	mlsum_instantiation_guide = """The languages supported for the Mlsum Dataset are:
	de - German
	es - Spanish
	fr - French
	ru - Russian
	tu - Turkish

	Examples to instantiate the dataset:
	1. Dataset with only one language
	dataset = MlsumDataset({language_token})
	dataset = MlsumDataset("es")
	dataset = MlsumDataset("tu")...

	2. Dataset with a multiple languages
	dataset = MlsumDataset({list of language_token})
	dataset = MlsumDataset(["es","de"])
	dataset = MlsumDataset(["es","de", "tu"])...

	3. Dataset with all supported languages (default)
	dataset = MlsumDataset(all)
	dataset = MlsumDataset()
	"""

	def __init__(self, languages: Optional[Union[str, List[str]]] = "all"):
	super().__init__(dataset_args=(languages,))

	def _load_dataset_safe(self, languages: Optional[Union[str, List[str]]]):
	"""
	Overrides the parent class method
	Method loads multiple datasets of different languages provided in :param languages:
	It then concatenates these datasets into one combined dataset
	:rtype: datasetDict containing the combined dataset
	:param languages: Optional, either a string or list of strings specifying the languages
	to load
	"""
	print(MlsumDataset.mlsum_instantiation_guide)

	# Choose languages to download articles
	if languages == "all":
	selected_languages = MlsumDataset.supported_languages
	elif isinstance(languages, list):
	for language in languages:
	assert self.is_supported(language)
	selected_languages = languages
	else:
	assert self.is_supported(languages)
	selected_languages = [languages]

	# Concatenate selected languaeges into one dataset
	language_datasets = []
	for language in selected_languages:
	dataset = super()._load_dataset_safe(
	"mlsum",
	language,
	)

	language_datasets.append(dataset)

	mlsum_dataset = self._concatenate_dataset_dicts(language_datasets)

	return mlsum_dataset

	def _process_data(self, data: Dataset) -> Generator[SummInstance, None, None]:
	"""
	Overrides the SummDataset '_process_data()' method
	This method processes the data contained in the dataset
	and puts each data instance into a SummInstance object
	:param dataset: a train/validation/test dataset
	:rtype: a generator yielding SummInstance objects
	"""
	for instance in tqdm(data):
	article: List = instance["text"]
	summary: str = instance["summary"]
	summ_instance = SummInstance(source=article, summary=summary)

	yield summ_instance

	def is_supported(self, language: str):
	"""
	Checks whether the requested langues is supported
	:param language: string containing the requested language
	:rtype bool:
	"""
	if language not in MlsumDataset.supported_languages:
	print(MlsumDataset.mlsum_instantiation_guide)
	raise ValueError(
	f"The language(s): '{language}' entered is not supported. See above message for usage info"
	)
	else:
	return True


	# Non-huggingface datasets


	class ScisummnetDataset(SummDataset):
	"""
	The SciSummNet dataset. As a dataset not included by huggingface, we need to do manually download, set basic
	information for the dataset
	"""

	dataset_name = "ScisummNet"

	version = "1.1.0"
	description = (
	"A summary of scientific papers should ideally incorporate the impact of the papers on the "
	"research community reflected by citations. To facilitate research in citation-aware scientific "
	"paper summarization (Scisumm), the CL-Scisumm shared task has been organized since 2014 for "
	"papers in the computational linguistics and NLP domain."
	)

	is_dialogue_based = False
	is_multi_document = False
	is_query_based = False

	huggingface_dataset = False
	builder_script_path = path.join(
	BASE_NONHUGGINGFACE_DATASETS_PATH, dataset_name.lower() + ".py"
	)

	def __init__(self, seed=None):
	super().__init__()

	def _process_data(self, data: Dataset) -> Generator[SummInstance, None, None]:
	"""
	Overrides the SummDataset '_process_data()' method
	This method processes the data contained in the dataset
	and puts each data instance into a SummInstance object
	:param dataset: a train/validation/test dataset
	:rtype: a generator yielding SummInstance objects
	"""
	for instance in tqdm(data):
	docs: List = [
	instance["document_xml"],
	instance["citing_sentences_annotated.json"],
	]
	summary: str = instance["summary"]
	summ_instance = SummInstance(source=docs, summary=summary)

	yield summ_instance


	class SummscreenDataset(SummDataset):
	"""
	The SummScreen dataset. As a dataset not included by huggingface, we need to do manually download, set basic
	information for the dataset
	"""

	dataset_name = "Summscreen"

	version = "1.1.0"
	is_dialogue_based = True
	is_multi_document = False
	is_query_based = False

	huggingface_dataset = False
	builder_script_path = path.join(
	BASE_NONHUGGINGFACE_DATASETS_PATH, dataset_name.lower() + ".py"
	)

	def __init__(self, seed=None):
	super().__init__()

	def _process_data(self, data: Dataset) -> Generator[SummInstance, None, None]:
	"""
	Overrides the SummDataset '_process_data()' method
	This method processes the data contained in the dataset
	and puts each data instance into a SummInstance object
	:param dataset: a train/validation/test dataset
	:rtype: a generator yielding SummInstance objects
	"""
	for instance in tqdm(data):
	transcript: List = instance[
	"transcript"
	] # convert string into a list of string dialogues
	recap: str = instance["recap"]
	summ_instance = SummInstance(source=transcript, summary=recap)

	yield summ_instance


	class QMsumDataset(SummDataset):
	"""
	QMSum Dataset
	"""

	dataset_name = "QMsum"
	description = """
	QMSum is a new human-annotated benchmark for query-based multi-domain meeting summarization task,
	which consists of 1,808 query-summary pairs over 232 meetings in multiple domains.
	"""

	is_dialogue_based = True
	is_multi_document = False
	is_query_based = True

	huggingface_dataset = False
	builder_script_path = path.join(
	BASE_NONHUGGINGFACE_DATASETS_PATH, dataset_name.lower() + ".py"
	)

	def __init__(self):
	super().__init__()

	def _process_data(self, data: Dataset) -> Generator[SummInstance, None, None]:
	"""
	Overrides the SummDataset '_process_data()' method
	This method processes the data contained in the dataset
	and puts each data instance into a SummInstance object
	:param dataset: a train/validation/test dataset
	:rtype: a generator yielding SummInstance objects
	"""
	for instance in tqdm(data):
	for query_set in (
	instance["general_query_list"] + instance["specific_query_list"]
	):
	meeting: List = [
	utterance["speaker"] + " : " + utterance["content"]
	for utterance in instance["meeting_transcripts"]
	]
	query: str = query_set["query"]
	summary: str = query_set["answer"]
	summ_instance = SummInstance(
	source=meeting, summary=summary, query=query
	)

	yield summ_instance


	class ArxivDataset(SummDataset):
	"""
	The Arxiv Dataset
	"""

	dataset_name = "Arxiv_longsummarization"
	description = """
	A summarization dataset comprised of pairs of scientific papers.
	The dataset provides a challenging testbed for abstractive summarization.
	It contains papers and their abstracts.
	"""

	is_dialogue_based = False
	is_multi_document = False
	is_query_based = False

	huggingface_dataset = False
	builder_script_path = path.join(
	BASE_NONHUGGINGFACE_DATASETS_PATH, dataset_name.lower() + ".py"
	)

	def __init__(self):

	print(
	"*****************",
	"*Attention*",
	"This dataset is quite large (approx 5Gb and will need about 15 Gb for the extraction process",
	"Cancel/interrupt the download if size and time constraints will not be met",
	"*****************",
	sep="\n",
	)

	super().__init__()

	def _process_data(self, data: Dataset) -> Generator[SummInstance, None, None]:
	"""
	Overrides the SummDataset '_process_data()' method
	This method processes the data contained in the dataset
	and puts each data instance into a SummInstance object
	:param dataset: a train/validation/test dataset
	:rtype: a generator yielding SummInstance objects
	"""
	for instance in tqdm(data):
	article: List = instance["article_text"]
	abstract: str = " ".join(instance["abstract_text"])
	summ_instance = SummInstance(source=article, summary=abstract)

	yield summ_instance