Repository: microsoft/promptflow
File: src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py (Python)

Code before:
    if evaluation_name is not None:
        if not isinstance(evaluation_name, str):
            raise ValueError("evaluation_name must be a string.")

    _validate_columns(data, evaluators, target)


def _validate_columns(data, evaluators, target):
    try:
        data_df = pd.read_json(data, lines=True)
    except Exception as e:
        raise ValueError(
            f"Failed to load data from {data}. Please validate it is a valid jsonl data. Error: {str(e)}.")

    if target:
        # If the target function is given, it may return
        # several columns and hence we cannot check the availability of columns
        # without knowing target function semantics.
        # Instead, here we will validate the columns, taken by target.
        _validate_input_data_for_evaluator(target, None, data_df, is_target_fn=True)
    else:
        for evaluator_name, evaluator in evaluators.items():
            _validate_input_data_for_evaluator(evaluator, evaluator_name, data_df)


def _apply_target_to_data(target: Callable, data: str, pf_client: PFClient) -> str:
    """
    Apply the target function to the data set and save the data to temporary file.

    :keyword target: The function to be applied to data.
    :paramtype target: Callable
    :keyword data: The path to input jsonl file.
    :paramtype data: str
    :keyword pf_client: The promptflow client to be used.
    :paramtype pf_client: PFClient
    :return: The path to data file with answers from target function.
    :rtype: str
    """
    with tempfile.TemporaryDirectory() as d:
        save_function_as_fow(fun=target, target_dir=d, pf=pf_client)
        run = pf_client.run(
            flow=d,
            data=data,
            name=f'preprocess_{uuid.uuid1()}'
        )
        run = pf_client.stream(run)
        function_output = pd.read_json(pf_client.runs._get_outputs_path(run),
                                       orient='records', lines=True)
    function_output.set_index(LINE_NUMBER, inplace=True)
    function_output.sort_index(inplace=True)
    data_input = pd.read_json(data, orient='records', lines=True)

Review comment (performance):
Any reason we don't batch run the target function first (base_run), and then call pf.run to evaluates that base_run? 

```
eval_run = pf.run(
    flow=eval_flow,
    data="./data.jsonl",  # path to the data file
    run=base_run,  # specify base_run as the run you want to evaluate
    column_mapping={
        "code": "${run.outputs.output}",
    },
    stream=True,
)
```

Code after:
        if not isinstance(evaluation_name, str):
            raise ValueError("evaluation_name must be a string.")

    try:
        initial_data_df = pd.read_json(data, lines=True)
    except Exception as e:
        raise ValueError(
            f"Failed to load data from {data}. Please validate it is a valid jsonl data. Error: {str(e)}.")

    _validate_columns(initial_data_df, evaluators, target)
    return initial_data_df


def _validate_columns(df: pd.DataFrame, evaluators: Dict[str, Any], target: Optional[Callable]) -> None:
    """
    Check that all columns needed by evaluator or target function are present.

    :keyword df: The data frame to be validated.
    :paramtype df: pd.DataFrame
    :keyword evaluators: The dictionary of evaluators.
    :paramtype evaluators: Dict[str, Any]
    :keyword target: The callable to be applied to data set.
    :paramtype target: Optional[Callable]
    """
    if target:
        # If the target function is given, it may return
        # several columns and hence we cannot check the availability of columns
        # without knowing target function semantics.
        # Instead, here we will validate the columns, taken by target.
        _validate_input_data_for_evaluator(target, None, df, is_target_fn=True)
    else:
        for evaluator_name, evaluator in evaluators.items():
            _validate_input_data_for_evaluator(evaluator, evaluator_name, df)


def _apply_target_to_data(
        target: Callable,
        data: str,
        pf_client: PFClient,
        initial_data: pd.DataFrame) -> Tuple[pd.DataFrame, Set[str]]:
    """
    Apply the target function to the data set and return updated data and generated columns.

    :keyword target: The function to be applied to data.
    :paramtype target: Callable
    :keyword data: The path to input jsonl file.
    :paramtype data: str
    :keyword pf_client: The promptflow client to be used.
    :paramtype pf_client: PFClient
    :keyword initial_data: The data frame with the loaded data.
    :paramtype initial_data: pd.DataFrame