Repository: microsoft/RD-Agent
File: rdagent/components/coder/data_science/share/eval.py (Python)

Code before:

        # Execute the main script
        stdout = remove_eda_part(
            implementation.execute(env=env, entry="strace -e trace=file -f -o trace.log python main.py --inference")
        )

        # walk model_folder and list the files
        model_folder_files = [
            str(file.relative_to(implementation.workspace_path)) for file in model_folder.iterdir() if file.is_file()
        ]

        opened_trace_lines = []
        if (implementation.workspace_path / "trace.log").exists():
            input_path = T("scenarios.data_science.share:scen.input_path").r()
            first_level_children = set()
            path_regex = re.compile(r'openat\(.+?,\s*"([^"]+)"')
            log_content = (implementation.workspace_path / "trace.log").read_text()

            for line in log_content.splitlines():
                if "openat" not in line or input_path not in line:
                    continue

                match = path_regex.search(line)
                if match:
                    full_path_str = match.group(1)
                    if full_path_str.startswith(input_path):
                        try:
                            relative_path = Path(full_path_str).relative_to(input_path)
                            if relative_path.parts:
                                first_level_children.add(relative_path.parts[0])
                        except ValueError:
                            continue

            opened_trace_lines = sorted(list(first_level_children))

        # this will assert the generation of necessary files
        for f in ["submission.csv", "scores.csv"]:
            if not (implementation.workspace_path / f).exists():
                err_msg = f"{f} does not exist. The model is not dumped. Make sure that the required files, like submission.csv and scores.csv, are created even if you bypass the model training step by loading the saved model file directly."
                return CoSTEERSingleFeedback(
                    execution=err_msg,
                    return_checking=err_msg,
                    code=err_msg,
                    final_decision=False,
                )

        # Check if scores contain NaN (values)
        score_df = pd.read_csv((implementation.workspace_path / "scores.csv"), index_col=0)
        if score_df.isnull().values.any():
            nan_locations = score_df[score_df.isnull().any(axis=1)]
            err_msg = f"\n[Error] The scores dataframe contains NaN values at the following locations:\n{nan_locations}"

Review comment (suggestion):
We should resolve (normalizing) the path before match it with each other.
Otherwise the match may not be  reliable.
<img width="1214" height="348" alt="image" src="https://github.com/user-attachments/assets/e5c88761-a662-4fcf-94d2-bd2fb731b61c" />


Code after:

        # Execute the main script
        stdout = remove_eda_part(
            implementation.execute(env=env, entry="strace -e trace=file -f -o trace.log python main.py --inference")
        )

        # walk model_folder and list the files
        model_folder_files = [
            str(file.relative_to(implementation.workspace_path)) for file in model_folder.iterdir() if file.is_file()
        ]

        opened_trace_lines = None
        if (implementation.workspace_path / "trace.log").exists():
            input_path = T("scenarios.data_science.share:scen.input_path").r()
            abs_input_path = str(Path(input_path).resolve())
            # matching path in string like `openat(AT_FDCWD, "/home/user/project/main.py", O_RDONLY) = 5`
            path_regex = re.compile(r'openat\(.+?,\s*"([^"]+)"')
            log_content = (implementation.workspace_path / "trace.log").read_text()

            opened_files = set()
            for line in log_content.splitlines():
                if "openat" not in line or (abs_input_path not in line and input_path not in line):
                    continue

                match = path_regex.search(line)
                if match:
                    full_path = Path(match.group(1)).resolve()
                    if str(full_path).startswith(abs_input_path):
                        opened_files.add(Path(data_source_path).resolve() / full_path.relative_to(abs_input_path))

            from rdagent.scenarios.data_science.scen.utils import FileTreeGenerator

            tree_gen = FileTreeGenerator(allowed_paths=opened_files)  # pass opened files filter
            opened_trace_lines = tree_gen.generate_tree(Path(data_source_path).resolve())
            # Limitation: training and test are expected to be different files.

        # this will assert the generation of necessary files
        for f in ["submission.csv", "scores.csv"]:
            if not (implementation.workspace_path / f).exists():
                err_msg = f"{f} does not exist. The model is not dumped. Make sure that the required files, like submission.csv and scores.csv, are created even if you bypass the model training step by loading the saved model file directly."
                return CoSTEERSingleFeedback(
                    execution=err_msg,
                    return_checking=err_msg,
                    code=err_msg,
                    final_decision=False,
                )

        # Check if scores contain NaN (values)
        score_df = pd.read_csv((implementation.workspace_path / "scores.csv"), index_col=0)
        if score_df.isnull().values.any():
            nan_locations = score_df[score_df.isnull().any(axis=1)]