Repository: vllm-project/vllm
File: vllm/v1/engine/core.py (Python)

Code before:
        self.scheduler.reset_encoder_cache()
        # Reset the GPU model runner's encoder cache (physical storage)
        self.model_executor.reset_encoder_cache()

    def pause_scheduler(
        self, mode: PauseMode = "abort", clear_cache: bool = True
    ) -> Future | None:
        """Pause generation; behavior depends on mode.

        All pause states queue new adds. PAUSE_ABORT and PAUSE_KEEP skip step();
        PAUSE_WAIT allows step() so in-flight requests can drain.

        - ``abort``: Set PAUSE_ABORT, abort all requests, wait for abort
          outputs to be sent (when running with output_queue), clear caches,
          then complete the returned Future.
        - ``wait``: Set PAUSE_WAIT (queue adds, keep stepping); when drained,
          set PAUSE_KEEP, clear caches, complete the returned Future.
        - ``keep``: Set PAUSE_KEEP; return a Future that completes when the
          output queue is empty.
        """
        if mode not in ("keep", "abort", "wait"):
            raise ValueError(f"Invalid pause mode: {mode}")

        output_queue = getattr(self, "output_queue", None)
        if output_queue is None and mode == "wait":
            raise ValueError("Cannot use 'wait' pause mode with inline-engine mode")

        def wait_until_idle(engine: "EngineCore", future: Future[Any] | None) -> bool:
            if engine.scheduler.has_unfinished_requests() or engine.batch_queue:
                return False
            if output_queue is not None and not output_queue.empty():
                return False
            if clear_cache:
                engine.reset_prefix_cache(reset_running_requests=True)
                engine.reset_mm_cache()
                engine.reset_encoder_cache()
            if future is not None:
                future.set_result(None)
            return True

        if mode == "abort":
            aborted_reqs = self.scheduler.finish_requests(
                None, RequestStatus.FINISHED_ABORTED
            )
            if output_queue is not None:
                self._send_abort_outputs(output_queue, aborted_reqs)

        pause_state = PauseState.PAUSED_ALL if mode == "keep" else PauseState.PAUSED_NEW
        self.scheduler.set_pause_state(pause_state)
        if wait_until_idle(self, future=None):
            return None

Review comment (suggestion):
I guess you made up "inline-engine mode" here because you're dissatisfied with the existing terminology, but I must say one of the things I found about the vLLM learning curve was how many different (new to me) terms were thrown around to mean the same thing ...

Here was Claude's take when I asked it:

>   I extrapolated "inline-engine mode" as a descriptive term for the InprocClient behavior, but it's not official terminology. The codebase primarily refers to it as:
>  - InprocClient (implementation class)
>  - in-process mode (descriptive)
>  - multiprocess_mode=False (configuration)


Code after:
        self.scheduler.reset_encoder_cache()
        # Reset the GPU model runner's encoder cache (physical storage)
        self.model_executor.reset_encoder_cache()

    def _reset_caches(self, reset_running_requests=True) -> None:
        self.reset_prefix_cache(reset_running_requests=reset_running_requests)
        self.reset_mm_cache()
        self.reset_encoder_cache()

    def pause_scheduler(
        self, mode: PauseMode = "abort", clear_cache: bool = True
    ) -> Future | None:
        """Pause generation; behavior depends on mode.

        All pause modes queue new adds -- "abort" and "keep" skip step();
        "wait" allows step() so in-flight requests can drain.

        - ``abort``: Set PAUSED_NEW, abort all requests, wait for abort
          outputs to be sent (when running with output_queue), optionally
          clear caches, then complete the returned Future.
        - ``wait``: Set PAUSED_NEW (queue adds, keep stepping); when drained,
          optionally clear caches, then complete the returned Future.
        - ``keep``: Set PAUSED_ALL; return a Future that completes when the
          output queue is empty.
        """
        if mode not in ("keep", "abort", "wait"):
            raise ValueError(f"Invalid pause mode: {mode}")
        if mode == "wait":
            raise ValueError("'wait' mode can't be used in inproc-engine mode")

        if mode == "abort":
            self.scheduler.finish_requests(None, RequestStatus.FINISHED_ABORTED)

        pause_state = PauseState.PAUSED_ALL if mode == "keep" else PauseState.PAUSED_NEW
        self.scheduler.set_pause_state(pause_state)
        if clear_cache:
            self._reset_caches()

        return None

    def resume_scheduler(self) -> None:
        """Resume the scheduler and flush any requests queued while paused."""
        self.scheduler.set_pause_state(PauseState.UNPAUSED)

    def is_scheduler_paused(self) -> bool:
        """Return whether the scheduler is in any pause state."""
        return self.scheduler.pause_state != PauseState.UNPAUSED

    def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None | Future:
        """Put the engine to sleep at the specified level.