Spaces:

transformersegmentation
/

segmentation_scores

Sleeping

segmentation_scores / segmentation_scores.py

Update segmentation_scores.py

313d779 about 1 year ago

12.5 kB

	# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	""" Segmentation scores evaluation metrics"""

	import evaluate
	import datasets


	# TODO: Add BibTeX citation
	_CITATION = """\
	@InProceedings{huggingface:module,
	title = {A great new module},
	authors={huggingface, Inc.},
	year={2020}
	}
	"""

	# TODO: Add description of the module here
	_DESCRIPTION = """\
	This module computes segmentation scores for a list of predicted segmentations and gold segmentations.
	"""


	# TODO: Add description of the arguments of the module here
	_KWARGS_DESCRIPTION = """
	Calculates how good are predicted segmentations, using boundary, token and type scores.
	Args:
	predictions: list of segmented utterances to score. Each predictions
	should be a string with phonemes separated by spaces and estimated word boundaries
	denoted by the token 'WORD_BOUNDARY'.
	references: list of segmented utterances to score. Each predictions
	should be a string with phonemes separated by spaces and gold word boundaries
	denoted by the token 'WORD_BOUNDARY'.
	Returns:
	type_fscore: lexicon f1 score
	type_precision: lexicon precision
	type_recall: lexicon recall
	token_fscore: token f1 score
	token_precision: token precision
	token_recall: token recall
	boundary_all_fscore: boundary f1 score, including utterance boundaries
	boundary_all_precision: boundary precision, including utterance boundaries
	boundary_all_recall: boundary recall, including utterance boundaries
	boundary_noedge_fscore: boundary f1 score, excluding utterance boundaries
	boundary_noedge_precision: boundary precision, excluding utterance boundaries
	boundary_noedge_recall: boundary recall, excluding utterance boundaries
	Examples:
	>>> segmentation_scores = evaluate.load("transformersegmentation/segmentation_scores")
	>>> results = segmentation_scores.compute(references=["w ɛ ɹ WORD_BOUNDARY ɪ z WORD_BOUNDARY ð ɪ s WORD_BOUNDARY", "l ɪ ɾ əl WORD_BOUNDARY aɪ z WORD_BOUNDARY"], predictions=["w ɛ ɹ WORD_BOUNDARY ɪ z WORD_BOUNDARY ð ɪ s WORD_BOUNDARY", "l ɪ ɾ əl WORD_BOUNDARY aɪ z WORD_BOUNDARY"])
	>>> print(results)
	{'type_fscore': 1.0, 'type_precision': 1.0, 'type_recall': 1.0, 'token_fscore': 1.0, 'token_precision': 1.0, 'token_recall': 1.0, 'boundary_all_fscore': 1.0, 'boundary_all_precision': 1.0, 'boundary_all_recall': 1.0, 'boundary_noedge_fscore': 1.0, 'boundary_noedge_precision': 1.0, 'boundary_noedge_recall': 1.0}
	"""

	class TokenEvaluation(object):
	"""Evaluation of token f-score, precision and recall"""

	def __init__(self):
	self.test = 0
	self.gold = 0
	self.correct = 0
	self.n = 0
	self.n_exactmatch = 0

	def precision(self):
	return float(self.correct) / self.test if self.test != 0 else None

	def recall(self):
	return float(self.correct) / self.gold if self.gold != 0 else None

	def fscore(self):
	total = self.test + self.gold
	return float(2 * self.correct) / total if total != 0 else None

	def exact_match(self):
	return float(self.n_exactmatch) / self.n if self.n else None

	def update(self, test_set, gold_set):
	self.n += 1

	if test_set == gold_set:
	self.n_exactmatch += 1

	# omit empty items for type scoring (should not affect token
	# scoring). Type lists are prepared with '_' where there is no
	# match, to keep list lengths the same
	self.test += len([x for x in test_set if x != "_"])
	self.gold += len([x for x in gold_set if x != "_"])
	self.correct += len(test_set & gold_set)

	def update_lists(self, test_sets, gold_sets):
	if len(test_sets) != len(gold_sets):
	raise ValueError(
	"#words different in test and gold: {} != {}".format(
	len(test_sets), len(gold_sets)
	)
	)

	for t, g in zip(test_sets, gold_sets):
	self.update(t, g)


	class TypeEvaluation(TokenEvaluation):
	"""Evaluation of type f-score, precision and recall"""

	@staticmethod
	def lexicon_check(textlex, goldlex):
	"""Compare hypothesis and gold lexicons"""
	textlist = []
	goldlist = []
	for w in textlex:
	if w in goldlex:
	# set up matching lists for the true positives
	textlist.append(w)
	goldlist.append(w)
	else:
	# false positives
	textlist.append(w)
	# ensure matching null element in text list
	goldlist.append("_")

	for w in goldlex:
	if w not in goldlist:
	# now for the false negatives
	goldlist.append(w)
	# ensure matching null element in text list
	textlist.append("_")

	textset = [{w} for w in textlist]
	goldset = [{w} for w in goldlist]
	return textset, goldset

	def update_lists(self, text, gold):
	lt, lg = self.lexicon_check(text, gold)
	super(TypeEvaluation, self).update_lists(lt, lg)


	class BoundaryEvaluation(TokenEvaluation):
	@staticmethod
	def get_boundary_positions(stringpos):
	return [{idx for pair in line for idx in pair} for line in stringpos]

	def update_lists(self, text, gold):
	lt = self.get_boundary_positions(text)
	lg = self.get_boundary_positions(gold)
	super(BoundaryEvaluation, self).update_lists(lt, lg)


	class BoundaryNoEdgeEvaluation(BoundaryEvaluation):
	@staticmethod
	def get_boundary_positions(stringpos):
	return [{left for left, _ in line if left > 0} for line in stringpos]


	class _StringPos(object):
	"""Compute start and stop index of words in an utterance"""

	def __init__(self):
	self.idx = 0

	def __call__(self, n):
	"""Return the position of the current word given its length `n`"""
	start = self.idx
	self.idx += n
	return start, self.idx



	@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
	class segmentation_scores(evaluate.Metric):
	"""TODO: Short description of my evaluation module."""

	def _info(self):
	# TODO: Specifies the evaluate.EvaluationModuleInfo object
	return evaluate.MetricInfo(
	# This is the description that will appear on the modules page.
	module_type="metric",
	description=_DESCRIPTION,
	citation=_CITATION,
	inputs_description=_KWARGS_DESCRIPTION,
	# This defines the format of each prediction and reference
	features=datasets.Features({
	'predictions': datasets.Value('string'),
	'references': datasets.Value('string'),
	}),
	# Homepage of the module for documentation
	homepage="https://huggingface.co/spaces/transformersegmentation/segmentation_scores",
	# Additional links to the codebase or references
	codebase_urls=["http://github.com/codebyzeb/transformersegmentation"],
	reference_urls=["http://path.to.reference.url/new_module"]
	)

	def _download_and_prepare(self, dl_manager):
	"""Optional: download external resources useful to compute the scores"""
	# TODO: Download external resources if needed
	pass

	def _process_data(self, text):
	""" Load text data for evaluation
	Parameters
	----------
	text : list of str
	The list of utterances to read for the evaluation.

	Returns
	-------
	(words, positions, lexicon) : three lists
	where `words` are the input utterances with word separators
	removed, `positions` stores the start/stop index of each word
	for each utterance, and `lexicon` is the list of words.
	"""
	words = []
	positions = []
	lexicon = {}

	# ignore empty lines
	for utt in (utt for utt in text if utt.strip()):
	# list of phones in the utterance with word seperator removed
	phone_in_utterance = [
	phone for phone in utt.split(" ") if phone != "WORD_BOUNDARY"
	]
	words_in_utterance = (
	"".join(
	" " if phone == "WORD_BOUNDARY" else phone for phone in utt.split(" ")
	)
	.strip()
	.split(" ")
	)

	words.append(phone_in_utterance)
	for word in words_in_utterance:
	lexicon[word] = 1
	idx = _StringPos()
	positions.append({idx(len(word)) for word in words_in_utterance})

	# return the words lexicon as a sorted list
	lexicon = sorted([k for k in lexicon.keys()])
	return words, positions, lexicon

	def _compute(self, predictions, references):
	"""Scores a segmented text against its gold version
	Parameters
	----------
	predictions : sequence of str
	A suite of word utterances, each string using 'WORD_BOUNDARY' as as word separator.
	references : sequence of str
	A suite of word utterances, each string using 'WORD_BOUNDARY' as as word separator.

	Returns
	-------
	scores : dict
	A dictionary with the following entries:
	* 'type_fscore'
	* 'type_precision'
	* 'type_recall'
	* 'token_fscore'
	* 'token_precision'
	* 'token_recall'
	* 'boundary_all_fscore'
	* 'boundary_all_precision'
	* 'boundary_all_recall'
	* 'boundary_noedge_fscore'
	* 'boundary_noedge_precision'
	* 'boundary_noedge_recall'

	Raises
	------
	ValueError
	If `gold` and `text` have different size or differ in tokens
	"""
	text_words, text_stringpos, text_lex = self._process_data(predictions)
	gold_words, gold_stringpos, gold_lex = self._process_data(references)

	if len(gold_words) != len(text_words):
	raise ValueError(
	"gold and train have different size: len(gold)={}, len(train)={}".format(
	len(gold_words), len(text_words)
	)
	)

	for i, (g, t) in enumerate(zip(gold_words, text_words)):
	if g != t:
	raise ValueError(
	'gold and train differ at line {}: gold="{}", train="{}"'.format(
	i + 1, g, t
	)
	)

	# token evaluation
	token_eval = TokenEvaluation()
	token_eval.update_lists(text_stringpos, gold_stringpos)

	# type evaluation
	type_eval = TypeEvaluation()
	type_eval.update_lists(text_lex, gold_lex)

	# boundary evaluation (with edges)
	boundary_eval = BoundaryEvaluation()
	boundary_eval.update_lists(text_stringpos, gold_stringpos)

	# boundary evaluation (no edges)
	boundary_noedge_eval = BoundaryNoEdgeEvaluation()
	boundary_noedge_eval.update_lists(text_stringpos, gold_stringpos)

	return {
	"token_precision": token_eval.precision(),
	"token_recall": token_eval.recall(),
	"token_fscore": token_eval.fscore(),
	"type_precision": type_eval.precision(),
	"type_recall": type_eval.recall(),
	"type_fscore": type_eval.fscore(),
	"boundary_all_precision": boundary_eval.precision(),
	"boundary_all_recall": boundary_eval.recall(),
	"boundary_all_fscore": boundary_eval.fscore(),
	"boundary_noedge_precision": boundary_noedge_eval.precision(),
	"boundary_noedge_recall": boundary_noedge_eval.recall(),
	"boundary_noedge_fscore": boundary_noedge_eval.fscore(),
	}