Repository: microsoft/RD-Agent
File: rdagent/scenarios/data_science/dev/runner/eval.py (Python)

Code before:
            time_spent_ratio < DS_RD_SETTING.time_ratio_limit_to_enable_hyperparameter_tuning
        ):
            enable_hyperparameter_tuning_check = True

        system_prompt = T(".prompts:DSCoSTEER_eval.system").r(
            scenario=self.scen.get_scenario_all_desc(eda_output=implementation.file_dict.get("EDA.md", None)),
            task_desc=target_task.get_task_information(),
            enable_hyperparameter_tuning_check=enable_hyperparameter_tuning_check,
        )
        user_prompt = T(".prompts:DSCoSTEER_eval.user").r(
            code=implementation.all_codes,
            change_summary=implementation.change_summary,
            stdout=shrink_text(stdout),
            time_spent=f"{implementation.running_info.running_time:.2f} seconds",
            timeout=f"{env.conf.running_timeout_period} seconds",
            percent_of_timeout_used=f"{time_spent_ratio * 100:.2f}%",
            queried_former_failed_knowledge=queried_former_failed_knowledge,
        )

        feedback = build_cls_from_json_with_retry(
            DSRunnerFeedback,
            system_prompt=system_prompt,
            user_prompt=user_prompt,
            # init_kwargs_update_func=DSRunnerFeedback.val_and_update_init_dict,
        )
        feedback.score = score_df.loc["ensemble"].iloc[0] if score_ret_code == 0 else None
        feedback.final_decision = feedback.acceptable and (
            not feedback.hyperparameter_tuning_decision
        )  # If hyperparameter_tuning_decision is None, it's considered as False, so the final_decision dependents on the acceptable

        if feedback and not DS_RD_SETTING.coder_on_whole_pipeline:
            # remove unused files
            implementation.execute(env=env, entry="python -m coverage json -o coverage.json")
            coverage_report_path = implementation.workspace_path / "coverage.json"
            if coverage_report_path.exists():
                used_files = set(json.loads(coverage_report_path.read_text())["files"].keys())
                coverage_report_path.unlink()
                logger.info(f"All used scripts: {used_files}")

                use_one_model = False
                for f in used_files:
                    if f.startswith("model_") and "test" not in f:
                        use_one_model = True
                        break

                if not use_one_model:
                    feedback.acceptable = feedback.final_decision = False
                    logger.warning("No model script is used in `main.py`.")
                    feedback.code += "\n[Error] No model script is used in `main.py`."

                all_python_files = set(Path(implementation.workspace_path).rglob("*.py"))

Review comment (suggestion):
we need to handle a corner case whether score_df doesn't have a "ensemble", this will raise error in this case

Code after:

        # 2. The current time spent on runner is less that the time_ratio_limit_to_enable_hyperparameter_tuning.
        time_spent_ratio = implementation.running_info.running_time / env.conf.running_timeout_period
        c2 = time_spent_ratio < DS_RD_SETTING.time_ratio_limit_to_enable_hyperparameter_tuning

        # 3. Only enable hyperparameter tuning during the merge stage if configured.
        if DS_RD_SETTING.only_enable_tuning_in_merge:
            c3 = timer.remain_time() >= timedelta(hours=DS_RD_SETTING.merge_hours)
        else:
            c3 = True

        # 4. If we set an overall hyperparameter tuning time ratio limit, only enable tuning if enough overall time remains.
        res_time = RD_Agent_TIMER_wrapper.timer.remain_time()
        total_time = RD_Agent_TIMER_wrapper.timer.all_duration
        res_ratio = res_time / total_time
        c4 = res_ratio >= DS_RD_SETTING.overall_time_ratio_limit_to_enable_hyperparameter_tuning

        # Only enable hyperparameter tuning check if all 4 criteria are met.
        enable_hyperparameter_tuning_check = c1 and c2 and c3 and c4

        system_prompt = T(".prompts:DSCoSTEER_eval.system").r(
            scenario=self.scen.get_scenario_all_desc(eda_output=implementation.file_dict.get("EDA.md", None)),
            task_desc=target_task.get_task_information(),
            enable_hyperparameter_tuning_check=enable_hyperparameter_tuning_check,
        )
        user_prompt = T(".prompts:DSCoSTEER_eval.user").r(
            code=implementation.all_codes,
            change_summary=implementation.change_summary,
            stdout=shrink_text(stdout),
            time_spent=f"{implementation.running_info.running_time:.2f} seconds",
            timeout=f"{env.conf.running_timeout_period} seconds",
            percent_of_timeout_used=f"{time_spent_ratio * 100:.2f}%",
            queried_former_failed_knowledge=queried_former_failed_knowledge,
        )

        feedback = build_cls_from_json_with_retry(
            DSRunnerFeedback,
            system_prompt=system_prompt,
            user_prompt=user_prompt,
            # init_kwargs_update_func=DSRunnerFeedback.val_and_update_init_dict,
        )
        try:
            feedback.score = score_df.loc["ensemble"].iloc[0] if score_ret_code == 0 else None
        except:
            logger.error("Failed to get the score from scores.csv.")
            feedback.score = None
        feedback.final_decision = feedback.acceptable and (
            not feedback.hyperparameter_tuning_decision
        )  # If hyperparameter_tuning_decision is None, it's considered as False, so the final_decision dependents on the acceptable

        if feedback and not DS_RD_SETTING.coder_on_whole_pipeline: