Repository: microsoft/promptflow
File: src/promptflow-evals/promptflow/evals/_common/rai_service.py (Python)

Code before:


def parse_response(  # pylint: disable=too-many-branches,too-many-statements
    batch_response: List[Dict], metric_name: str
) -> Dict:
    """Parse the annotation response from Responsible AI service for a content harm evaluation.

    :param batch_response: The annotation response from Responsible AI service.
    :type batch_response: List[Dict]
    :param metric_name: The evaluation metric to use.
    :type metric_name: str
    :return: The parsed annotation result.
    :rtype: List[List[Dict]]
    """
    # non-numeric metrics
    if metric_name in {EvaluationMetrics.PROTECTED_MATERIAL, _InternalEvaluationMetrics.ECI}:
        if not batch_response or len(batch_response[0]) == 0 or metric_name not in batch_response[0]:
            return {}
        response = batch_response[0][metric_name]
        response = response.replace("false", "False")
        response = response.replace("true", "True")
        parsed_response = literal_eval(response)
        result = {}
        metric_prefix = _get_metric_prefix(metric_name)
        # Use label instead of score since these are assumed to be boolean results.
        result[metric_prefix + "_label"] = parsed_response["label"] if "label" in parsed_response else np.nan
        result[metric_prefix + "_reasoning"] = parsed_response["reasoning"] if "reasoning" in parsed_response else ""
        return result
    return _parse_content_harm_response(batch_response, metric_name)


def _get_metric_prefix(metric_name: str) -> str:
    """Get the prefix for the evaluation metric. This is usually the metric name.

    :param metric_name: The evaluation metric to use.
    :type metric_name: str
    :return: The prefix for the evaluation metric.
    :rtype: str
    """
    if metric_name == _InternalEvaluationMetrics.ECI:
        return "ECI"
    return metric_name


def _parse_content_harm_response(batch_response: List[Dict], metric_name: str) -> Dict:
    """Parse the annotation response from Responsible AI service for a content harm evaluation.

    :param batch_response: The annotation response from Responsible AI service.
    :type batch_response: List[Dict]
    :param metric_name: The evaluation metric to use.
    :type metric_name: str

Review comment (suggestion):
`np.nan` is float type. I wonder if we should just return an empty string.

Code after:


def parse_response(  # pylint: disable=too-many-branches,too-many-statements
    batch_response: List[Dict], metric_name: str
) -> Dict:
    """Parse the annotation response from Responsible AI service for a content harm evaluation.

    :param batch_response: The annotation response from Responsible AI service.
    :type batch_response: List[Dict]
    :param metric_name: The evaluation metric to use.
    :type metric_name: str
    :return: The parsed annotation result.
    :rtype: List[List[Dict]]
    """
    # non-numeric metrics
    if metric_name in {EvaluationMetrics.PROTECTED_MATERIAL, _InternalEvaluationMetrics.ECI}:
        if not batch_response or len(batch_response[0]) == 0 or metric_name not in batch_response[0]:
            return {}
        response = batch_response[0][metric_name]
        response = response.replace("false", "False")
        response = response.replace("true", "True")
        parsed_response = literal_eval(response)
        result = {}
        metric_prefix = _get_metric_prefix(metric_name)
        # Use label instead of score since these are assumed to be boolean results.
        result[metric_prefix + "_label"] = parsed_response["label"] if "label" in parsed_response else ""
        result[metric_prefix + "_reasoning"] = parsed_response["reasoning"] if "reasoning" in parsed_response else ""
        return result
    return _parse_content_harm_response(batch_response, metric_name)


def _get_metric_prefix(metric_name: str) -> str:
    """Get the prefix for the evaluation metric. This is usually the metric name.

    :param metric_name: The evaluation metric to use.
    :type metric_name: str
    :return: The prefix for the evaluation metric.
    :rtype: str
    """
    if metric_name == _InternalEvaluationMetrics.ECI:
        return "ECI"
    return metric_name


def _parse_content_harm_response(batch_response: List[Dict], metric_name: str) -> Dict:
    """Parse the annotation response from Responsible AI service for a content harm evaluation.

    :param batch_response: The annotation response from Responsible AI service.
    :type batch_response: List[Dict]
    :param metric_name: The evaluation metric to use.
    :type metric_name: str