Repository: ray-project/ray
File: python/ray/tune/search/basic_variant.py (Python)

Code before:
    ):
        if trial_id in self._live_trials:
            self._live_trials.remove(trial_id)

    def get_state(self):
        if any(iterator.lazy_eval for iterator in self._iterators):
            return False
        state = self.__dict__.copy()
        del state["_trial_generator"]
        return state

    def set_state(self, state):
        self.__dict__.update(state)
        for iterator in self._iterators:
            self._trial_generator = itertools.chain(self._trial_generator, iterator)

    def save_to_dir(self, dirpath, session_str):
        if any(iterator.lazy_eval for iterator in self._iterators):
            return False
        state_dict = self.get_state()
        file_name = self.CKPT_FILE_TMPL.format(session_str)
        _atomic_save(
            state=state_dict,
            checkpoint_dir=dirpath,
            file_name=file_name,
            tmp_file_name=f".tmp-{file_name}",
        )

    def has_checkpoint(self, dirpath: str):
        """Whether a checkpoint file exists within dirpath."""
        return any(Path(dirpath).glob(self.CKPT_FILE_TMPL.format("*")))

    def restore_from_dir(self, dirpath: str):
        """Restores self + searcher + search wrappers from dirpath."""
        state_dict = _load_newest_checkpoint(dirpath, self.CKPT_FILE_TMPL.format("*"))
        if not state_dict:
            raise RuntimeError("Unable to find checkpoint in {}.".format(dirpath))
        self.set_state(state_dict)


Review comment (suggestion):
It's not clear to me if this solves the problem.
1. The previous file names (e.g. `.tmp_generator`, `.tmp_search_generator_ckpt`, etc.) are different from each other so there should not be any contention there.
2. The repro is only running a single job so the uniqueness of `session_str` in `file_name` doesn't make a difference.

Code after:
    ):
        if trial_id in self._live_trials:
            self._live_trials.remove(trial_id)

    def get_state(self):
        if any(iterator.lazy_eval for iterator in self._iterators):
            return False
        state = self.__dict__.copy()
        del state["_trial_generator"]
        return state

    def set_state(self, state):
        self.__dict__.update(state)
        for iterator in self._iterators:
            self._trial_generator = itertools.chain(self._trial_generator, iterator)

    def save_to_dir(self, dirpath, session_str):
        if any(iterator.lazy_eval for iterator in self._iterators):
            return False
        state_dict = self.get_state()
        file_name = self.CKPT_FILE_TMPL.format(session_str)
        _atomic_save(
            state=state_dict,
            checkpoint_dir=dirpath,
            file_name=file_name,
            tmp_file_name=f"tmp-{file_name}",
        )

    def has_checkpoint(self, dirpath: str):
        """Whether a checkpoint file exists within dirpath."""
        return any(Path(dirpath).glob(self.CKPT_FILE_TMPL.format("*")))

    def restore_from_dir(self, dirpath: str):
        """Restores self + searcher + search wrappers from dirpath."""
        state_dict = _load_newest_checkpoint(dirpath, self.CKPT_FILE_TMPL.format("*"))
        if not state_dict:
            raise RuntimeError("Unable to find checkpoint in {}.".format(dirpath))
        self.set_state(state_dict)