Repository: google/mediapipe
File: mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm (Objective-C++)

Code before:
      CGImageRef cgImageRef = [MPPCGImageUtils cgImageFromImageFrame:image.GetImageFrameSharedPtr()
                                                 shouldCopyPixelData:shouldCopyPixelData
                                                               error:error];
      UIImage *image = [UIImage imageWithCGImage:cgImageRef];
      CGImageRelease(cgImageRef);

      return [self initWithUIImage:image orientation:sourceImage.orientation error:nil];
    }
    case MPPImageSourceTypePixelBuffer: {
      if (!shouldCopyPixelData) {
        // TODO: Investigate possibility of permuting channels of `mediapipe::Image` returned by
        // vision tasks in place to ensure that we can support creating `CVPixelBuffer`s without
        // copying the pixel data.
        [MPPCommonUtils
            createCustomError:error
                     withCode:MPPTasksErrorCodeInvalidArgumentError
                  description:
                      @"When the source type is pixel buffer, you cannot request uncopied data."];
        return nil;
      }

      CVPixelBufferRef pixelBuffer =
          [MPPCVPixelBufferUtils cvPixelBufferFromImageFrame:*(image.GetImageFrameSharedPtr())
                                                       error:error];
      MPPImage *image = [self initWithPixelBuffer:pixelBuffer
                                      orientation:sourceImage.orientation
                                            error:nil];
      CVPixelBufferRelease(pixelBuffer);
      return image;
    }
    case MPPImageSourceTypeSampleBuffer: {
      if (!shouldCopyPixelData) {
        // TODO: Investigate possibility of permuting channels of `mediapipe::Image` returned by
        // vision tasks in place to ensure that we can support creating `CVPixelBuffer`s without
        // copying the pixel data.
        [MPPCommonUtils
            createCustomError:error
                     withCode:MPPTasksErrorCodeInvalidArgumentError
                  description:
                      @"When the source type is sample buffer, you cannot request uncopied data."];
        return nil;
      }

      CVPixelBufferRef pixelBuffer =
          [MPPCVPixelBufferUtils cvPixelBufferFromImageFrame:*(image.GetImageFrameSharedPtr())
                                                       error:error];

      CMSampleTimingInfo timingInfo;
      if (CMSampleBufferGetSampleTimingInfo(sourceImage.sampleBuffer, 0, &timingInfo) != 0) {
        [MPPCommonUtils createCustomError:error
                                 withCode:MPPTasksErrorCodeInvalidArgumentError

Review comment (performance):
@schmidt-sebastian Wrapping the inference calls inside loops is something we can leave to the users too in scenarios they need to. Normally this is done from within loops when you observe your apps temporary objects are not getting destroyed. Since this is a feature of UIImages, it's not really a memory leak we create. But a lot Swift devs maybe not be aware of such a thing. We don't really know how the APIs are going to be used. So choosing to do it here just to be safe.

Not needed for the other source types (pixel buffer, sample buffer). Their native create methods don't return autoreleased objects. Thus memory gets deallocated as expected.

Code after:
                             withCode:MPPTasksErrorCodeInvalidArgumentError
                          description:@"Source image cannot be nil."];
    return nil;
  }

  switch (sourceImage.imageSourceType) {
    case MPPImageSourceTypeImage: {
      CGImageRef cgImageRef = [MPPCGImageUtils cgImageFromImageFrame:image.GetImageFrameSharedPtr()
                                                 shouldCopyPixelData:shouldCopyPixelData
                                                               error:error];

      // `[UIImage imageWithCGImage]` seems to be returning an autoreleased object. Thus ARC only
      // deallocates it when the autoreleasepool to which the image was added is drained. This may
      // happen only much later during the life cycle of the app. For a standalone inference this
      // isn't a concern. If this method is invoked in a loop, the unreleased UIImage's accumulate
      // in memory. They get destroyed all at once when all iterations of the loop are completed.
      // For infinite loops like the camera callback, this results in an increase in memory
      // footprint over time. To avoid this, the`UIImage` is being created in an @autoreleasepool
      // block which results in the image being released as soon as the block completes. The MPImage
      // retains the UIImage, to keep the image alive during its lifetime.
      // (Until caller keeps a reference to the result retured by the task)
      //
      // Reference: `Use Local Autorelease Pool Blocks to Reduce Peak Memory Footprint`
      // https://developer.apple.com/library/archive/documentation/Cocoa/Conceptual/MemoryMgmt/Articles/mmAutoreleasePools.html.
      MPPImage *mpImage;
      @autoreleasepool {
        UIImage *uiImage = [UIImage imageWithCGImage:cgImageRef];
        mpImage = [self initWithUIImage:uiImage orientation:sourceImage.orientation error:nil];
        CGImageRelease(cgImageRef);
      }

      return mpImage;
    }
    case MPPImageSourceTypePixelBuffer: {
      if (!shouldCopyPixelData) {
        // TODO: Investigate possibility of permuting channels of `mediapipe::Image` returned by
        // vision tasks in place to ensure that we can support creating `CVPixelBuffer`s without
        // copying the pixel data.
        [MPPCommonUtils
            createCustomError:error
                     withCode:MPPTasksErrorCodeInvalidArgumentError
                  description:
                      @"When the source type is pixel buffer, you cannot request uncopied data."];
        return nil;
      }

      CVPixelBufferRef pixelBuffer =
          [MPPCVPixelBufferUtils cvPixelBufferFromImageFrame:*(image.GetImageFrameSharedPtr())
                                                       error:error];
      MPPImage *mpImage = [self initWithPixelBuffer:pixelBuffer
                                        orientation:sourceImage.orientation