Repository: qdrant/qdrant
File: lib/collection/src/update_workers/optimization_worker.rs (Rust)

Code before:
    where
        F: Fn() + Send + Clone + Sync + 'static,
    {
        let mut handles = vec![];
        let is_optimization_failed = Arc::new(AtomicBool::new(false));

        let scheduled;
        let mut pending_segments = 0;
        let mut pending_points = 0;
        {
            let segments = segments.read();
            let mut planner = OptimizationPlanner::new(
                segments.running_optimizations.count(),
                segments.iter_original(),
            );
            for optimizer in optimizers.iter() {
                planner.set_optimizer(Arc::clone(optimizer));
                optimizer.plan_optimizations(&mut planner);
            }
            scheduled = planner.into_scheduled();

            for (_, segment_ids) in &scheduled {
                pending_segments += segment_ids.len();
                for &segment_id in segment_ids {
                    if let Some(LockedSegment::Original(segment)) = segments.get(segment_id) {
                        pending_points += segment.read().total_point_count()
                    }
                }
            }
        }

        optimizers_log.lock().pending = PendingOptimizations {
            optimizations: scheduled.len(),
            segments: pending_segments,
            points: pending_points,
        };

        for (optimizer, segments_to_merge) in scheduled {
            let Some(optimizer) = optimizer else {
                debug_assert!(false);
                continue;
            };

            // Return early if we reached the optimization job limit
            if limit.map(|extra| handles.len() >= extra).unwrap_or(false) {
                log::trace!("Reached optimization job limit, postponing other optimizations");
                break;
            }

            // If optimization failed, we should not endlessly try to optimize same segments
            if is_optimization_failed.load(Ordering::Relaxed) {

Review comment (performance):
Or `available_point_count`? If the pending optimization is a vacuum optimizer, the `pending_points` may be much larger than the result after optimization

Code after:
    ) -> Vec<StoppableTaskHandle<bool>>
    where
        F: Fn() + Send + Clone + Sync + 'static,
    {
        let mut handles = vec![];
        let is_optimization_failed = Arc::new(AtomicBool::new(false));

        let scheduled;
        let mut pending_segments = 0;
        let mut pending_points = 0;
        {
            let segments = segments.read();
            let mut planner = OptimizationPlanner::new(
                segments.running_optimizations.count(),
                segments.iter_original(),
            );
            for optimizer in optimizers.iter() {
                planner.set_optimizer(Arc::clone(optimizer));
                optimizer.plan_optimizations(&mut planner);
            }
            scheduled = planner.into_scheduled();

            for (_, segment_ids) in &scheduled {
                pending_segments += segment_ids.len();
                for &segment_id in segment_ids {
                    if let Some(LockedSegment::Original(segment)) = segments.get(segment_id) {
                        pending_points += segment.read().available_point_count();
                    }
                }
            }
        }

        optimizers_log.lock().pending = PendingOptimizations {
            optimizations: scheduled.len(),
            segments: pending_segments,
            points: pending_points,
        };

        for (optimizer, segments_to_merge) in scheduled {
            let Some(optimizer) = optimizer else {
                debug_assert!(false);
                continue;
            };

            // Return early if we reached the optimization job limit
            if limit.map(|extra| handles.len() >= extra).unwrap_or(false) {
                log::trace!("Reached optimization job limit, postponing other optimizations");
                break;
            }

            // If optimization failed, we should not endlessly try to optimize same segments