Repository: ray-project/ray
File: python/ray/serve/_private/request_router/request_router.py (Python)

Code before:
        Uses dict index for O(1) lookup by multiplexed_model_id.
        Also performs lazy cleanup of done futures to prevent memory leaks.
        """
        if request_metadata is None or not request_metadata.multiplexed_model_id:
            return None

        model_id = request_metadata.multiplexed_model_id
        candidates = self._pending_requests_by_model_id.get(model_id)
        if candidates is None:
            return None

        # Clean done entries from front using O(1) popleft.
        # Since fulfillment is roughly FIFO, done entries accumulate at the front.
        while candidates and candidates[0].future.done():
            candidates.popleft()

        if len(candidates) == 0:
            # Clean up empty deques to prevent memory leak.
            self._pending_requests_by_model_id.pop(model_id, None)
            return None

        # Find first non-done request. Usually at front after cleanup,
        # but there may be done entries in the middle from out-of-order fulfillment.
        for pr in candidates:
            if not pr.future.done():
                return pr

        return None

    def _update_multiplexed_model_ids_with_replicas(
        self, replicas: List[RunningReplica]
    ):
        """Update the multiplexed model IDs based on the replicas.

        This should be called when the replicas are updated.
        """
        new_multiplexed_model_id_to_replica_ids = defaultdict(set)

        for r in replicas:
            for model_id in r.multiplexed_model_ids:
                new_multiplexed_model_id_to_replica_ids[model_id].add(r.replica_id)

        self._multiplexed_model_id_to_replica_ids = (
            new_multiplexed_model_id_to_replica_ids
        )

    def _get_replica_ids_with_fewest_multiplexed_models(self) -> Set[str]:
        """Get the set of replicas that have the fewest multiplexed models loaded."""
        candidates = set()
        sorted_replicas = sorted(
            self._replicas.values(), key=lambda x: len(x.multiplexed_model_ids)

Review comment (suggestion):
would this simplification work?
```suggestion
        while candidates:
            pr = candidates[0]
            if not pr.future.done():
                return pr
            else:
                candidates.popleft()
                
        self._pending_requests_by_model_id.pop(model_id, None)
```

Code after:
        Uses dict index for O(1) lookup by multiplexed_model_id.
        Also performs lazy cleanup of done futures to prevent memory leaks.
        """
        if request_metadata is None or not request_metadata.multiplexed_model_id:
            return None

        model_id = request_metadata.multiplexed_model_id
        candidates = self._pending_requests_by_model_id.get(model_id)
        while candidates:
            pr = candidates[0]
            if not pr.future.done():
                return pr
            candidates.popleft()
        self._pending_requests_by_model_id.pop(model_id, None)
        return None

    def _update_multiplexed_model_ids_with_replicas(
        self, replicas: List[RunningReplica]
    ):
        """Update the multiplexed model IDs based on the replicas.

        This should be called when the replicas are updated.
        """
        new_multiplexed_model_id_to_replica_ids = defaultdict(set)

        for r in replicas:
            for model_id in r.multiplexed_model_ids:
                new_multiplexed_model_id_to_replica_ids[model_id].add(r.replica_id)

        self._multiplexed_model_id_to_replica_ids = (
            new_multiplexed_model_id_to_replica_ids
        )

    def _get_replica_ids_with_fewest_multiplexed_models(self) -> Set[str]:
        """Get the set of replicas that have the fewest multiplexed models loaded."""
        candidates = set()
        sorted_replicas = sorted(
            self._replicas.values(), key=lambda x: len(x.multiplexed_model_ids)
        )
        least_num_multiplexed_model_ids = math.inf
        for replica in sorted_replicas:
            if len(replica.multiplexed_model_ids) <= least_num_multiplexed_model_ids:
                candidates.add(replica.replica_id)
                least_num_multiplexed_model_ids = len(replica.multiplexed_model_ids)
            else:
                break

        return candidates

    @property
    def _multiplexed_matching_timeout(self) -> float: