Spaces:
Runtime error
Runtime error
File size: 2,407 Bytes
129cd69 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 |
import re
from typing import Any, List
from langchain.evaluation.schema import StringEvaluator
class RegexMatchStringEvaluator(StringEvaluator):
"""Compute a regex match between the prediction and the reference.
Examples
----------
>>> evaluator = RegexMatchStringEvaluator(flags=re.IGNORECASE)
>>> evaluator.evaluate_strings(
prediction="Mindy is the CTO",
reference="^mindy.*cto$",
) # This will return {'score': 1.0} due to the IGNORECASE flag
>>> evaluator = RegexMatchStringEvaluator()
>>> evaluator.evaluate_strings(
prediction="Mindy is the CTO",
reference="^Mike.*CEO$",
) # This will return {'score': 0.0}
>>> evaluator.evaluate_strings(
prediction="Mindy is the CTO",
reference="^Mike.*CEO$|^Mindy.*CTO$",
) # This will return {'score': 1.0} as the prediction matches the second pattern in the union
""" # noqa: E501
def __init__(self, *, flags: int = 0, **kwargs: Any): # Default is no flags
super().__init__()
self.flags = flags
@property
def requires_input(self) -> bool:
"""
This evaluator does not require input.
"""
return False
@property
def requires_reference(self) -> bool:
"""
This evaluator requires a reference.
"""
return True
@property
def input_keys(self) -> List[str]:
"""
Get the input keys.
Returns:
List[str]: The input keys.
"""
return ["reference", "prediction"]
@property
def evaluation_name(self) -> str:
"""
Get the evaluation name.
Returns:
str: The evaluation name.
"""
return "regex_match"
def _evaluate_strings( # type: ignore[arg-type,override]
self,
*,
prediction: str,
reference: str,
**kwargs: Any,
) -> dict:
"""
Evaluate the regex match between the prediction and the reference.
Args:
prediction (str): The prediction string.
reference (Optional[str], optional): The reference regex pattern.
Returns:
dict: The evaluation results containing the score.
"""
match = re.match(reference, prediction, flags=self.flags)
return {"score": int(bool(match))}
|