File size: 5,316 Bytes
b783bce e5a087b 6f3c593 c4dd600 e5a087b f32487c b783bce e5a087b 6798e06 e5a087b b783bce e5a087b b783bce e5a087b b783bce f61d613 b783bce e5a087b c4dd600 e5a087b b783bce f32487c b783bce e5a087b b783bce f32487c b783bce e5a087b 6798e06 b783bce 6452fbf e5a087b 5818152 e5a087b b783bce e5a087b f61d613 468c32e f61d613 f32487c e5a087b 6798e06 c4dd600 6798e06 c4dd600 6798e06 c4dd600 6798e06 c4dd600 6798e06 e5a087b f95da7e 5818152 e5a087b 5818152 e5a087b 6798e06 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 |
from dataclasses import field
from typing import Any, Dict, Generator, Iterable, List, Optional, Union
import datasets
import evaluate
from datasets import Features, Sequence, Value
from .artifact import __file__ as _
from .blocks import __file__ as _
from .card import __file__ as _
from .catalog import __file__ as _
from .collections import __file__ as _
from .common import __file__ as _
from .dataclass import __file__ as _
from .dict_utils import __file__ as _
from .file_utils import __file__ as _
from .formats import __file__ as _
from .fusion import __file__ as _
from .generator_utils import __file__ as _
from .hf_utils import __file__ as _
from .instructions import __file__ as _
from .load import __file__ as _
from .loaders import __file__ as _
from .metrics import __file__ as _
from .normalizers import __file__ as _
from .operator import (
MultiStreamOperator,
SequntialOperator,
SequntialOperatorInitilizer,
StreamInitializerOperator,
)
from .operator import __file__ as _
from .operators import (
ApplyOperatorsField,
ApplyStreamOperatorsField,
FlattenInstances,
MergeStreams,
SplitByValue,
)
from .operators import __file__ as _
from .processors import __file__ as _
from .random_utils import __file__ as _
from .recipe import __file__ as _
from .register import __file__ as _
from .register import register_all_artifacts
from .renderers import __file__ as _
from .schema import __file__ as _
from .split_utils import __file__ as _
from .splitters import __file__ as _
from .standard import __file__ as _
from .stream import MultiStream, Stream
from .stream import __file__ as _
from .task import __file__ as _
from .templates import __file__ as _
from .text_utils import __file__ as _
from .type_utils import __file__ as _
from .utils import __file__ as _
from .validate import __file__ as _
from .version import __file__ as _
class MultiStreamScoreMean(MultiStreamOperator):
def aggegate_results(self, multi_stream: MultiStream):
scores = []
for stream in multi_stream.values():
instance = stream.peak()
scores.append(instance["score"]["global"]["score"])
from statistics import mean
return mean(scores)
def spread_results(self, stream: Stream, score: float):
for instance in stream:
instance["score"]["global"]["groups_mean_score"] = score
yield instance
def process(self, multi_stream: MultiStream) -> MultiStream:
mean_score = self.aggegate_results(multi_stream)
result = {}
for stream_name, stream in multi_stream.items():
result[stream_name] = Stream(self.spread_results, gen_kwargs={"stream": stream, "score": mean_score})
return MultiStream(result)
class FromPredictionsAndOriginalData(StreamInitializerOperator):
def zip(self, predictions, references):
for prediction, original in zip(predictions, references):
yield {**original, "prediction": prediction}
def process(self, predictions: List[str], references: Iterable, split_name: str = "all") -> MultiStream:
return MultiStream(
{split_name: Stream(self.zip, gen_kwargs={"predictions": predictions, "references": references})}
)
from .schema import UNITXT_DATASET_SCHEMA
class MetricRecipe(SequntialOperatorInitilizer):
def prepare(self):
register_all_artifacts()
self.steps = [
FromPredictionsAndOriginalData(),
ApplyOperatorsField(
inputs_fields=["prediction", "references"],
fields_to_treat_as_list=["references"],
operators_field="postprocessors",
default_operators=["processors.to_string_stripped"],
),
SplitByValue(["group"]),
ApplyStreamOperatorsField(
"metrics",
reversed=True,
),
MultiStreamScoreMean(),
MergeStreams(),
]
UNITXT_METRIC_SCHEMA = Features({"predictions": Value("string"), "references": dict(UNITXT_DATASET_SCHEMA)})
def _compute(predictions: List[str], references: Iterable, flatten: bool = False, split_name: str = "all"):
recipe = MetricRecipe()
multi_stream = recipe(predictions=predictions, references=references, split_name=split_name)
if flatten:
operator = FlattenInstances()
multi_stream = operator(multi_stream)
stream = multi_stream[split_name]
return list(stream)
# TODO: currently we have two classes with this name. metric.Metric and matrics.Metric...
# @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
class Metric(evaluate.Metric):
def _info(self):
return evaluate.MetricInfo(
description="_DESCRIPTION",
citation="_CITATION",
# inputs_description=_KWARGS_DESCRIPTION,
features=UNITXT_METRIC_SCHEMA,
codebase_urls=["https://"],
reference_urls=[
"https://",
"https://",
],
)
def _compute(self, predictions: List[str], references: Iterable, flatten: bool = False, split_name: str = "all"):
return _compute(predictions=predictions, references=references, flatten=flatten, split_name=split_name)
|