Spaces:
Runtime error
Runtime error
# coding=utf-8 | |
# Copyright 2022 Meta Platforms authors and HuggingFace Inc. | |
# | |
# Licensed under the Apache License, Version 2.0 (the "License"); | |
# you may not use this file except in compliance with the License. | |
# You may obtain a copy of the License at | |
# | |
# http://www.apache.org/licenses/LICENSE-2.0 | |
# | |
# Unless required by applicable law or agreed to in writing, software | |
# distributed under the License is distributed on an "AS IS" BASIS, | |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
# See the License for the specific language governing permissions and | |
# limitations under the License. | |
import random | |
import unittest | |
import numpy as np | |
from transformers.testing_utils import require_torch, require_vision | |
from transformers.utils import is_torch_available, is_vision_available | |
from ...test_image_processing_common import ImageProcessingTestMixin, prepare_image_inputs | |
if is_torch_available(): | |
import torch | |
if is_vision_available(): | |
import PIL | |
from transformers import FlavaImageProcessor | |
from transformers.image_utils import PILImageResampling | |
from transformers.models.flava.image_processing_flava import ( | |
FLAVA_CODEBOOK_MEAN, | |
FLAVA_CODEBOOK_STD, | |
FLAVA_IMAGE_MEAN, | |
FLAVA_IMAGE_STD, | |
) | |
else: | |
FLAVA_IMAGE_MEAN = FLAVA_IMAGE_STD = FLAVA_CODEBOOK_MEAN = FLAVA_CODEBOOK_STD = None | |
class FlavaImageProcessingTester(unittest.TestCase): | |
def __init__( | |
self, | |
parent, | |
batch_size=7, | |
num_channels=3, | |
min_resolution=30, | |
max_resolution=400, | |
do_resize=True, | |
size=None, | |
do_center_crop=True, | |
crop_size=None, | |
resample=None, | |
do_rescale=True, | |
rescale_factor=1 / 255, | |
do_normalize=True, | |
image_mean=FLAVA_IMAGE_MEAN, | |
image_std=FLAVA_IMAGE_STD, | |
input_size_patches=14, | |
total_mask_patches=75, | |
mask_group_max_patches=None, | |
mask_group_min_patches=16, | |
mask_group_min_aspect_ratio=0.3, | |
mask_group_max_aspect_ratio=None, | |
codebook_do_resize=True, | |
codebook_size=None, | |
codebook_resample=None, | |
codebook_do_center_crop=True, | |
codebook_crop_size=None, | |
codebook_do_map_pixels=True, | |
codebook_do_normalize=True, | |
codebook_image_mean=FLAVA_CODEBOOK_MEAN, | |
codebook_image_std=FLAVA_CODEBOOK_STD, | |
): | |
size = size if size is not None else {"height": 224, "width": 224} | |
crop_size = crop_size if crop_size is not None else {"height": 224, "width": 224} | |
codebook_size = codebook_size if codebook_size is not None else {"height": 112, "width": 112} | |
codebook_crop_size = codebook_crop_size if codebook_crop_size is not None else {"height": 112, "width": 112} | |
self.parent = parent | |
self.batch_size = batch_size | |
self.num_channels = num_channels | |
self.do_resize = do_resize | |
self.do_rescale = do_rescale | |
self.rescale_factor = rescale_factor | |
self.min_resolution = min_resolution | |
self.max_resolution = max_resolution | |
self.size = size | |
self.resample = resample if resample is not None else PILImageResampling.BICUBIC | |
self.do_normalize = do_normalize | |
self.image_mean = image_mean | |
self.image_std = image_std | |
self.do_center_crop = do_center_crop | |
self.crop_size = crop_size | |
self.input_size_patches = input_size_patches | |
self.total_mask_patches = total_mask_patches | |
self.mask_group_max_patches = mask_group_max_patches | |
self.mask_group_min_patches = mask_group_min_patches | |
self.mask_group_min_aspect_ratio = mask_group_min_aspect_ratio | |
self.mask_group_max_aspect_ratio = mask_group_max_aspect_ratio | |
self.codebook_do_resize = codebook_do_resize | |
self.codebook_size = codebook_size | |
self.codebook_resample = codebook_resample if codebook_resample is not None else PILImageResampling.LANCZOS | |
self.codebook_do_center_crop = codebook_do_center_crop | |
self.codebook_crop_size = codebook_crop_size | |
self.codebook_do_map_pixels = codebook_do_map_pixels | |
self.codebook_do_normalize = codebook_do_normalize | |
self.codebook_image_mean = codebook_image_mean | |
self.codebook_image_std = codebook_image_std | |
def prepare_image_processor_dict(self): | |
return { | |
"image_mean": self.image_mean, | |
"image_std": self.image_std, | |
"do_normalize": self.do_normalize, | |
"do_resize": self.do_resize, | |
"size": self.size, | |
"resample": self.resample, | |
"do_rescale": self.do_rescale, | |
"rescale_factor": self.rescale_factor, | |
"do_center_crop": self.do_center_crop, | |
"crop_size": self.crop_size, | |
"input_size_patches": self.input_size_patches, | |
"total_mask_patches": self.total_mask_patches, | |
"mask_group_max_patches": self.mask_group_max_patches, | |
"mask_group_min_patches": self.mask_group_min_patches, | |
"mask_group_min_aspect_ratio": self.mask_group_min_aspect_ratio, | |
"mask_group_max_aspect_ratio": self.mask_group_min_aspect_ratio, | |
"codebook_do_resize": self.codebook_do_resize, | |
"codebook_size": self.codebook_size, | |
"codebook_resample": self.codebook_resample, | |
"codebook_do_center_crop": self.codebook_do_center_crop, | |
"codebook_crop_size": self.codebook_crop_size, | |
"codebook_do_map_pixels": self.codebook_do_map_pixels, | |
"codebook_do_normalize": self.codebook_do_normalize, | |
"codebook_image_mean": self.codebook_image_mean, | |
"codebook_image_std": self.codebook_image_std, | |
} | |
def get_expected_image_size(self): | |
return (self.size["height"], self.size["width"]) | |
def get_expected_mask_size(self): | |
return ( | |
(self.input_size_patches, self.input_size_patches) | |
if not isinstance(self.input_size_patches, tuple) | |
else self.input_size_patches | |
) | |
def get_expected_codebook_image_size(self): | |
return (self.codebook_size["height"], self.codebook_size["width"]) | |
def prepare_image_inputs(self, equal_resolution=False, numpify=False, torchify=False): | |
return prepare_image_inputs( | |
batch_size=self.batch_size, | |
num_channels=self.num_channels, | |
min_resolution=self.min_resolution, | |
max_resolution=self.max_resolution, | |
equal_resolution=equal_resolution, | |
numpify=numpify, | |
torchify=torchify, | |
) | |
class FlavaImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase): | |
image_processing_class = FlavaImageProcessor if is_vision_available() else None | |
maxDiff = None | |
def setUp(self): | |
self.image_processor_tester = FlavaImageProcessingTester(self) | |
def image_processor_dict(self): | |
return self.image_processor_tester.prepare_image_processor_dict() | |
def test_image_processor_properties(self): | |
image_processing = self.image_processing_class(**self.image_processor_dict) | |
self.assertTrue(hasattr(image_processing, "image_mean")) | |
self.assertTrue(hasattr(image_processing, "image_std")) | |
self.assertTrue(hasattr(image_processing, "do_normalize")) | |
self.assertTrue(hasattr(image_processing, "do_resize")) | |
self.assertTrue(hasattr(image_processing, "resample")) | |
self.assertTrue(hasattr(image_processing, "crop_size")) | |
self.assertTrue(hasattr(image_processing, "do_center_crop")) | |
self.assertTrue(hasattr(image_processing, "do_rescale")) | |
self.assertTrue(hasattr(image_processing, "rescale_factor")) | |
self.assertTrue(hasattr(image_processing, "masking_generator")) | |
self.assertTrue(hasattr(image_processing, "codebook_do_resize")) | |
self.assertTrue(hasattr(image_processing, "codebook_size")) | |
self.assertTrue(hasattr(image_processing, "codebook_resample")) | |
self.assertTrue(hasattr(image_processing, "codebook_do_center_crop")) | |
self.assertTrue(hasattr(image_processing, "codebook_crop_size")) | |
self.assertTrue(hasattr(image_processing, "codebook_do_map_pixels")) | |
self.assertTrue(hasattr(image_processing, "codebook_do_normalize")) | |
self.assertTrue(hasattr(image_processing, "codebook_image_mean")) | |
self.assertTrue(hasattr(image_processing, "codebook_image_std")) | |
def test_image_processor_from_dict_with_kwargs(self): | |
image_processor = self.image_processing_class.from_dict(self.image_processor_dict) | |
self.assertEqual(image_processor.size, {"height": 224, "width": 224}) | |
self.assertEqual(image_processor.crop_size, {"height": 224, "width": 224}) | |
self.assertEqual(image_processor.codebook_size, {"height": 112, "width": 112}) | |
self.assertEqual(image_processor.codebook_crop_size, {"height": 112, "width": 112}) | |
image_processor = self.image_processing_class.from_dict( | |
self.image_processor_dict, size=42, crop_size=84, codebook_size=33, codebook_crop_size=66 | |
) | |
self.assertEqual(image_processor.size, {"height": 42, "width": 42}) | |
self.assertEqual(image_processor.crop_size, {"height": 84, "width": 84}) | |
self.assertEqual(image_processor.codebook_size, {"height": 33, "width": 33}) | |
self.assertEqual(image_processor.codebook_crop_size, {"height": 66, "width": 66}) | |
def test_call_pil(self): | |
# Initialize image_processing | |
image_processing = self.image_processing_class(**self.image_processor_dict) | |
# create random PIL images | |
image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False) | |
for image in image_inputs: | |
self.assertIsInstance(image, PIL.Image.Image) | |
# Test not batched input | |
encoded_images = image_processing(image_inputs[0], return_tensors="pt") | |
# Test no bool masked pos | |
self.assertFalse("bool_masked_pos" in encoded_images) | |
expected_height, expected_width = self.image_processor_tester.get_expected_image_size() | |
self.assertEqual( | |
encoded_images.pixel_values.shape, | |
(1, self.image_processor_tester.num_channels, expected_height, expected_width), | |
) | |
# Test batched | |
encoded_images = image_processing(image_inputs, return_tensors="pt") | |
expected_height, expected_width = self.image_processor_tester.get_expected_image_size() | |
# Test no bool masked pos | |
self.assertFalse("bool_masked_pos" in encoded_images) | |
self.assertEqual( | |
encoded_images.pixel_values.shape, | |
( | |
self.image_processor_tester.batch_size, | |
self.image_processor_tester.num_channels, | |
expected_height, | |
expected_width, | |
), | |
) | |
def _test_call_framework(self, instance_class, prepare_kwargs): | |
# Initialize image_processing | |
image_processing = self.image_processing_class(**self.image_processor_dict) | |
# create random tensors | |
image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, **prepare_kwargs) | |
for image in image_inputs: | |
self.assertIsInstance(image, instance_class) | |
# Test not batched input | |
encoded_images = image_processing(image_inputs[0], return_tensors="pt") | |
expected_height, expected_width = self.image_processor_tester.get_expected_image_size() | |
self.assertEqual( | |
encoded_images.pixel_values.shape, | |
(1, self.image_processor_tester.num_channels, expected_height, expected_width), | |
) | |
encoded_images = image_processing(image_inputs, return_image_mask=True, return_tensors="pt") | |
expected_height, expected_width = self.image_processor_tester.get_expected_image_size() | |
self.assertEqual( | |
encoded_images.pixel_values.shape, | |
( | |
self.image_processor_tester.batch_size, | |
self.image_processor_tester.num_channels, | |
expected_height, | |
expected_width, | |
), | |
) | |
expected_height, expected_width = self.image_processor_tester.get_expected_mask_size() | |
self.assertEqual( | |
encoded_images.bool_masked_pos.shape, | |
( | |
self.image_processor_tester.batch_size, | |
expected_height, | |
expected_width, | |
), | |
) | |
# Test batched | |
encoded_images = image_processing(image_inputs, return_tensors="pt").pixel_values | |
expected_height, expected_width = self.image_processor_tester.get_expected_image_size() | |
self.assertEqual( | |
encoded_images.shape, | |
( | |
self.image_processor_tester.batch_size, | |
self.image_processor_tester.num_channels, | |
expected_height, | |
expected_width, | |
), | |
) | |
# Test masking | |
encoded_images = image_processing(image_inputs, return_image_mask=True, return_tensors="pt") | |
expected_height, expected_width = self.image_processor_tester.get_expected_image_size() | |
self.assertEqual( | |
encoded_images.pixel_values.shape, | |
( | |
self.image_processor_tester.batch_size, | |
self.image_processor_tester.num_channels, | |
expected_height, | |
expected_width, | |
), | |
) | |
expected_height, expected_width = self.image_processor_tester.get_expected_mask_size() | |
self.assertEqual( | |
encoded_images.bool_masked_pos.shape, | |
( | |
self.image_processor_tester.batch_size, | |
expected_height, | |
expected_width, | |
), | |
) | |
def test_call_numpy(self): | |
self._test_call_framework(np.ndarray, prepare_kwargs={"numpify": True}) | |
def test_call_numpy_4_channels(self): | |
self.image_processing_class.num_channels = 4 | |
self._test_call_framework(np.ndarray, prepare_kwargs={"numpify": True}) | |
self.image_processing_class.num_channels = 3 | |
def test_call_pytorch(self): | |
self._test_call_framework(torch.Tensor, prepare_kwargs={"torchify": True}) | |
def test_masking(self): | |
# Initialize image_processing | |
random.seed(1234) | |
image_processing = self.image_processing_class(**self.image_processor_dict) | |
image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, torchify=True) | |
# Test not batched input | |
encoded_images = image_processing(image_inputs[0], return_image_mask=True, return_tensors="pt") | |
self.assertEqual(encoded_images.bool_masked_pos.sum().item(), 75) | |
def test_codebook_pixels(self): | |
# Initialize image_processing | |
image_processing = self.image_processing_class(**self.image_processor_dict) | |
# create random PIL images | |
image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False) | |
for image in image_inputs: | |
self.assertIsInstance(image, PIL.Image.Image) | |
# Test not batched input | |
encoded_images = image_processing(image_inputs[0], return_codebook_pixels=True, return_tensors="pt") | |
expected_height, expected_width = self.image_processor_tester.get_expected_codebook_image_size() | |
self.assertEqual( | |
encoded_images.codebook_pixel_values.shape, | |
(1, self.image_processor_tester.num_channels, expected_height, expected_width), | |
) | |
# Test batched | |
encoded_images = image_processing(image_inputs, return_codebook_pixels=True, return_tensors="pt") | |
expected_height, expected_width = self.image_processor_tester.get_expected_codebook_image_size() | |
self.assertEqual( | |
encoded_images.codebook_pixel_values.shape, | |
( | |
self.image_processor_tester.batch_size, | |
self.image_processor_tester.num_channels, | |
expected_height, | |
expected_width, | |
), | |
) | |