haipingwu commited on
Commit
6ff8b35
1 Parent(s): b330978

fix import

Browse files
Files changed (2) hide show
  1. image_processing_phi3_v.py +0 -274
  2. processing_phi3_v.py +262 -1
image_processing_phi3_v.py DELETED
@@ -1,274 +0,0 @@
1
- # coding=utf-8
2
- # Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
3
- #
4
- # Licensed under the Apache License, Version 2.0 (the "License");
5
- # you may not use this file except in compliance with the License.
6
- # You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS,
12
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- # See the License for the specific language governing permissions and
14
- # limitations under the License.
15
-
16
- """Image processor class for Phi3-V."""
17
-
18
- from typing import List, Optional, Union
19
-
20
- import numpy as np
21
-
22
- from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
23
- from transformers.image_transforms import (
24
- convert_to_rgb,
25
- )
26
- from transformers.image_utils import (
27
- OPENAI_CLIP_MEAN,
28
- OPENAI_CLIP_STD,
29
- ImageInput,
30
- make_list_of_images,
31
- valid_images,
32
- )
33
- from transformers.utils import TensorType, is_vision_available, logging
34
-
35
- from transformers import AutoImageProcessor
36
-
37
- logger = logging.get_logger(__name__)
38
-
39
-
40
- if is_vision_available():
41
- from PIL import Image
42
-
43
- import torch
44
- import torchvision
45
-
46
- def padding_336(b):
47
- width, height = b.size
48
- tar = int(np.ceil(height / 336) * 336)
49
- top_padding = int((tar - height)/2)
50
- bottom_padding = tar - height - top_padding
51
- left_padding = 0
52
- right_padding = 0
53
- b = torchvision.transforms.functional.pad(b, [left_padding, top_padding, right_padding, bottom_padding], fill=[255,255,255])
54
-
55
- return b
56
-
57
- def calc_padded_size(width, height, padding_unit=336):
58
- target_height = int(np.ceil(height / padding_unit) * padding_unit)
59
- top_padding = int((target_height - height) / 2)
60
- bottom_padding = target_height - height - top_padding
61
- left_padding = 0
62
- right_padding = 0
63
- padded_width = width + left_padding + right_padding
64
- padded_height = height + top_padding + bottom_padding
65
- return padded_width, padded_height
66
-
67
- def HD_transform(img, hd_num=16):
68
- width, height = img.size
69
- trans = False
70
- if width < height:
71
- img = img.transpose(Image.TRANSPOSE)
72
- trans = True
73
- width, height = img.size
74
- ratio = (width/ height)
75
- scale = 1
76
- while scale*np.ceil(scale/ratio) <= hd_num:
77
- scale += 1
78
- scale -= 1
79
- new_w = int(scale * 336)
80
- new_h = int(new_w / ratio)
81
-
82
- img = torchvision.transforms.functional.resize(img, [new_h, new_w],)
83
- img = padding_336(img)
84
- width, height = img.size
85
- if trans:
86
- img = img.transpose(Image.TRANSPOSE)
87
-
88
- return img
89
-
90
- def calc_hd_transform_size(width, height, hd_num=16):
91
- transposed = False
92
- if width < height:
93
- width, height = height, width
94
- transposed = True
95
-
96
- ratio = width / height
97
- scale = 1
98
- while scale * np.ceil(scale / ratio) <= hd_num:
99
- scale += 1
100
- scale -= 1
101
-
102
- new_width = int(scale * 336)
103
- new_height = int(new_width / ratio)
104
-
105
- padded_width, padded_height = calc_padded_size(new_width, new_height)
106
-
107
- if transposed:
108
- padded_width, padded_height = padded_height, padded_width
109
-
110
- return padded_width, padded_height
111
-
112
- def pad_to_max_num_crops_tensor(images, max_crops=5):
113
- """
114
- images: B x 3 x H x W, B<=max_crops
115
- """
116
- B, _, H, W = images.shape
117
- if B < max_crops:
118
- pad = torch.zeros(max_crops - B, 3, H, W, dtype=images.dtype, device=images.device)
119
- images = torch.cat([images, pad], dim=0)
120
- return images
121
-
122
-
123
- class Phi3VImageProcessor(BaseImageProcessor):
124
- r"""
125
- Constructs a Phi3 image processor. Based on [`CLIPImageProcessor`] with incorporation of additional techniques
126
- for processing high resolution images as explained in the [InternLM-XComposer2-4KHD](https://arxiv.org/pdf/2404.06512)
127
-
128
- Args:
129
- image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`):
130
- Mean to use if normalizing the image. This is a float or list of floats the length of the number of
131
- channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method.
132
- image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`):
133
- Standard deviation to use if normalizing the image. This is a float or list of floats the length of the
134
- number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method.
135
- Can be overridden by the `image_std` parameter in the `preprocess` method.
136
- do_convert_rgb (`bool`, *optional*, defaults to `True`):
137
- Whether to convert the image to RGB.
138
- """
139
-
140
- model_input_names = ["pixel_values"]
141
-
142
- def __init__(
143
- self,
144
- num_crops: int = 1,
145
- image_mean: Optional[Union[float, List[float]]] = None,
146
- image_std: Optional[Union[float, List[float]]] = None,
147
- do_convert_rgb: bool = True,
148
- **kwargs,
149
- ) -> None:
150
- super().__init__(**kwargs)
151
- self.num_crops = num_crops
152
- self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN
153
- self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD
154
- self.do_convert_rgb = do_convert_rgb
155
-
156
- def calc_num_image_tokens(
157
- self,
158
- images: ImageInput
159
- ):
160
- """ Calculate the number of image tokens for each image.
161
- Args:
162
- images (`ImageInput`):
163
- Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
164
- passing in images with pixel values between 0 and 1, set `do_rescale=False`.
165
- """
166
- images = make_list_of_images(images)
167
-
168
- if not valid_images(images):
169
- raise ValueError(
170
- "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
171
- "torch.Tensor, tf.Tensor or jax.ndarray."
172
- )
173
-
174
- images = [image.convert('RGB') for image in images]
175
- # (H, W, C)
176
- elems = [HD_transform(im, hd_num = self.num_crops) for im in images]
177
- shapes = [[im.size[1], im.size[0]] for im in elems]
178
- num_img_tokens = [int((h//336*w//336+1)*144 + 1 + (h//336+1)*12) for h, w in shapes]
179
- return num_img_tokens
180
-
181
- def calc_num_image_tokens_from_image_size(self, width, height):
182
- """
183
- Calculate the number of image tokens for a given image size.
184
- Args:
185
- width (`int`): Width of the image.
186
- height (`int`): Height of the image.
187
- """
188
- new_width, new_height = calc_hd_transform_size(width, height, hd_num=self.num_crops)
189
- num_img_tokens = int((new_height // 336 * new_width // 336 + 1) * 144 + 1 + (new_height // 336 + 1) * 12)
190
- return num_img_tokens
191
-
192
- def preprocess(
193
- self,
194
- images: ImageInput,
195
- image_mean: Optional[Union[float, List[float]]] = None,
196
- image_std: Optional[Union[float, List[float]]] = None,
197
- do_convert_rgb: bool = None,
198
- return_tensors: Optional[Union[str, TensorType]] = None,
199
- ):
200
- """
201
- Args:
202
- images (`ImageInput`):
203
- Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
204
- passing in images with pixel values between 0 and 1, set `do_rescale=False`.
205
- image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
206
- Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
207
- image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
208
- Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
209
- `True`.
210
- do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
211
- Whether to convert the image to RGB.
212
- return_tensors (`str` or `TensorType`, *optional*):
213
- The type of tensors to return. Can be one of:
214
- - Unset: Return a list of `np.ndarray`.
215
- - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
216
- - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
217
- - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
218
- - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
219
- """
220
- image_mean = image_mean if image_mean is not None else self.image_mean
221
- image_std = image_std if image_std is not None else self.image_std
222
- do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
223
-
224
- images = make_list_of_images(images)
225
-
226
- if not valid_images(images):
227
- raise ValueError(
228
- "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
229
- "torch.Tensor, tf.Tensor or jax.ndarray."
230
- )
231
-
232
- if do_convert_rgb:
233
- images = [convert_to_rgb(image) for image in images]
234
-
235
- image_sizes = []
236
- img_processor = torchvision.transforms.Compose([
237
- torchvision.transforms.ToTensor(),
238
- torchvision.transforms.Normalize(image_mean, image_std)
239
- ])
240
-
241
- # PIL images
242
- # HD_transform pad images to size of multiiply of 336, 336
243
- # convert to RGB first
244
- images = [image.convert('RGB') for image in images]
245
- elems = [HD_transform(im, hd_num = self.num_crops) for im in images]
246
- # tensor transform and normalize
247
- hd_images = [img_processor(im) for im in elems]
248
- # create global image
249
- global_image = [torch.nn.functional.interpolate(im.unsqueeze(0).float(), size=(336, 336), mode='bicubic',).to(im.dtype) for im in hd_images]
250
-
251
- # [(3, h, w)], where h, w is multiple of 336
252
- shapes = [[im.size(1), im.size(2)] for im in hd_images]
253
- num_img_tokens = [int(((h//336)*(w//336)+1)*144 + 1 + (h//336+1)*12) for h, w in shapes]
254
- # reshape to channel dimension -> (num_images, num_crops, 3, 336, 336)
255
- # (1, 3, h//336, 336, w//336, 336) -> (1, h//336, w//336, 3, 336, 336) -> (h//336*w//336, 3, 336, 336)
256
- hd_images_reshape = [im.reshape(1, 3, h//336, 336, w//336, 336).permute(0,2,4,1,3,5).reshape(-1, 3, 336, 336).contiguous() for im, (h, w) in zip(hd_images, shapes)]
257
- # concat global image and local image
258
- hd_images_reshape = [torch.cat([_global_image] + [_im], dim=0) for _global_image, _im in zip(global_image, hd_images_reshape)]
259
-
260
- # pad to max_num_crops
261
- image_transformed = [pad_to_max_num_crops_tensor(im, self.num_crops+1) for im in hd_images_reshape]
262
- image_transformed = torch.stack(image_transformed, dim=0)
263
- image_sizes = [torch.LongTensor(_shapes) for _shapes in shapes]
264
- padded_images = image_transformed
265
- image_sizes = shapes
266
-
267
- data = {"pixel_values": padded_images,
268
- "image_sizes": image_sizes,
269
- "num_img_tokens": num_img_tokens
270
- }
271
-
272
- return BatchFeature(data=data, tensor_type=return_tensors)
273
-
274
- AutoImageProcessor.register("Phi3VImageProcessor", Phi3VImageProcessor)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
processing_phi3_v.py CHANGED
@@ -27,7 +27,268 @@ from transformers.image_utils import ImageInput
27
  from transformers.processing_utils import ProcessorMixin
28
  from transformers.tokenization_utils_base import PaddingStrategy, TextInput, TruncationStrategy
29
  from transformers.utils import TensorType
30
- from .image_processing_phi3_v import Phi3VImageProcessor
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31
  transformers.Phi3VImageProcessor = Phi3VImageProcessor
32
 
33
  class Phi3VProcessor(ProcessorMixin):
 
27
  from transformers.processing_utils import ProcessorMixin
28
  from transformers.tokenization_utils_base import PaddingStrategy, TextInput, TruncationStrategy
29
  from transformers.utils import TensorType
30
+
31
+
32
+ """Image processor class for Phi3-V."""
33
+
34
+ from typing import List, Optional, Union
35
+
36
+ import numpy as np
37
+
38
+ from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
39
+ from transformers.image_transforms import (
40
+ convert_to_rgb,
41
+ )
42
+ from transformers.image_utils import (
43
+ OPENAI_CLIP_MEAN,
44
+ OPENAI_CLIP_STD,
45
+ ImageInput,
46
+ make_list_of_images,
47
+ valid_images,
48
+ )
49
+ from transformers.utils import TensorType, is_vision_available, logging
50
+
51
+ from transformers import AutoImageProcessor
52
+
53
+ logger = logging.get_logger(__name__)
54
+
55
+
56
+ if is_vision_available():
57
+ from PIL import Image
58
+
59
+ import torch
60
+ import torchvision
61
+
62
+ def padding_336(b):
63
+ width, height = b.size
64
+ tar = int(np.ceil(height / 336) * 336)
65
+ top_padding = int((tar - height)/2)
66
+ bottom_padding = tar - height - top_padding
67
+ left_padding = 0
68
+ right_padding = 0
69
+ b = torchvision.transforms.functional.pad(b, [left_padding, top_padding, right_padding, bottom_padding], fill=[255,255,255])
70
+
71
+ return b
72
+
73
+ def calc_padded_size(width, height, padding_unit=336):
74
+ target_height = int(np.ceil(height / padding_unit) * padding_unit)
75
+ top_padding = int((target_height - height) / 2)
76
+ bottom_padding = target_height - height - top_padding
77
+ left_padding = 0
78
+ right_padding = 0
79
+ padded_width = width + left_padding + right_padding
80
+ padded_height = height + top_padding + bottom_padding
81
+ return padded_width, padded_height
82
+
83
+ def HD_transform(img, hd_num=16):
84
+ width, height = img.size
85
+ trans = False
86
+ if width < height:
87
+ img = img.transpose(Image.TRANSPOSE)
88
+ trans = True
89
+ width, height = img.size
90
+ ratio = (width/ height)
91
+ scale = 1
92
+ while scale*np.ceil(scale/ratio) <= hd_num:
93
+ scale += 1
94
+ scale -= 1
95
+ new_w = int(scale * 336)
96
+ new_h = int(new_w / ratio)
97
+
98
+ img = torchvision.transforms.functional.resize(img, [new_h, new_w],)
99
+ img = padding_336(img)
100
+ width, height = img.size
101
+ if trans:
102
+ img = img.transpose(Image.TRANSPOSE)
103
+
104
+ return img
105
+
106
+ def calc_hd_transform_size(width, height, hd_num=16):
107
+ transposed = False
108
+ if width < height:
109
+ width, height = height, width
110
+ transposed = True
111
+
112
+ ratio = width / height
113
+ scale = 1
114
+ while scale * np.ceil(scale / ratio) <= hd_num:
115
+ scale += 1
116
+ scale -= 1
117
+
118
+ new_width = int(scale * 336)
119
+ new_height = int(new_width / ratio)
120
+
121
+ padded_width, padded_height = calc_padded_size(new_width, new_height)
122
+
123
+ if transposed:
124
+ padded_width, padded_height = padded_height, padded_width
125
+
126
+ return padded_width, padded_height
127
+
128
+ def pad_to_max_num_crops_tensor(images, max_crops=5):
129
+ """
130
+ images: B x 3 x H x W, B<=max_crops
131
+ """
132
+ B, _, H, W = images.shape
133
+ if B < max_crops:
134
+ pad = torch.zeros(max_crops - B, 3, H, W, dtype=images.dtype, device=images.device)
135
+ images = torch.cat([images, pad], dim=0)
136
+ return images
137
+
138
+
139
+ class Phi3VImageProcessor(BaseImageProcessor):
140
+ r"""
141
+ Constructs a Phi3 image processor. Based on [`CLIPImageProcessor`] with incorporation of additional techniques
142
+ for processing high resolution images as explained in the [InternLM-XComposer2-4KHD](https://arxiv.org/pdf/2404.06512)
143
+
144
+ Args:
145
+ image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`):
146
+ Mean to use if normalizing the image. This is a float or list of floats the length of the number of
147
+ channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method.
148
+ image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`):
149
+ Standard deviation to use if normalizing the image. This is a float or list of floats the length of the
150
+ number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method.
151
+ Can be overridden by the `image_std` parameter in the `preprocess` method.
152
+ do_convert_rgb (`bool`, *optional*, defaults to `True`):
153
+ Whether to convert the image to RGB.
154
+ """
155
+
156
+ model_input_names = ["pixel_values"]
157
+
158
+ def __init__(
159
+ self,
160
+ num_crops: int = 1,
161
+ image_mean: Optional[Union[float, List[float]]] = None,
162
+ image_std: Optional[Union[float, List[float]]] = None,
163
+ do_convert_rgb: bool = True,
164
+ **kwargs,
165
+ ) -> None:
166
+ super().__init__(**kwargs)
167
+ self.num_crops = num_crops
168
+ self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN
169
+ self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD
170
+ self.do_convert_rgb = do_convert_rgb
171
+
172
+ def calc_num_image_tokens(
173
+ self,
174
+ images: ImageInput
175
+ ):
176
+ """ Calculate the number of image tokens for each image.
177
+ Args:
178
+ images (`ImageInput`):
179
+ Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
180
+ passing in images with pixel values between 0 and 1, set `do_rescale=False`.
181
+ """
182
+ images = make_list_of_images(images)
183
+
184
+ if not valid_images(images):
185
+ raise ValueError(
186
+ "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
187
+ "torch.Tensor, tf.Tensor or jax.ndarray."
188
+ )
189
+
190
+ images = [image.convert('RGB') for image in images]
191
+ # (H, W, C)
192
+ elems = [HD_transform(im, hd_num = self.num_crops) for im in images]
193
+ shapes = [[im.size[1], im.size[0]] for im in elems]
194
+ num_img_tokens = [int((h//336*w//336+1)*144 + 1 + (h//336+1)*12) for h, w in shapes]
195
+ return num_img_tokens
196
+
197
+ def calc_num_image_tokens_from_image_size(self, width, height):
198
+ """
199
+ Calculate the number of image tokens for a given image size.
200
+ Args:
201
+ width (`int`): Width of the image.
202
+ height (`int`): Height of the image.
203
+ """
204
+ new_width, new_height = calc_hd_transform_size(width, height, hd_num=self.num_crops)
205
+ num_img_tokens = int((new_height // 336 * new_width // 336 + 1) * 144 + 1 + (new_height // 336 + 1) * 12)
206
+ return num_img_tokens
207
+
208
+ def preprocess(
209
+ self,
210
+ images: ImageInput,
211
+ image_mean: Optional[Union[float, List[float]]] = None,
212
+ image_std: Optional[Union[float, List[float]]] = None,
213
+ do_convert_rgb: bool = None,
214
+ return_tensors: Optional[Union[str, TensorType]] = None,
215
+ ):
216
+ """
217
+ Args:
218
+ images (`ImageInput`):
219
+ Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
220
+ passing in images with pixel values between 0 and 1, set `do_rescale=False`.
221
+ image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
222
+ Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
223
+ image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
224
+ Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
225
+ `True`.
226
+ do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
227
+ Whether to convert the image to RGB.
228
+ return_tensors (`str` or `TensorType`, *optional*):
229
+ The type of tensors to return. Can be one of:
230
+ - Unset: Return a list of `np.ndarray`.
231
+ - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
232
+ - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
233
+ - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
234
+ - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
235
+ """
236
+ image_mean = image_mean if image_mean is not None else self.image_mean
237
+ image_std = image_std if image_std is not None else self.image_std
238
+ do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
239
+
240
+ images = make_list_of_images(images)
241
+
242
+ if not valid_images(images):
243
+ raise ValueError(
244
+ "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
245
+ "torch.Tensor, tf.Tensor or jax.ndarray."
246
+ )
247
+
248
+ if do_convert_rgb:
249
+ images = [convert_to_rgb(image) for image in images]
250
+
251
+ image_sizes = []
252
+ img_processor = torchvision.transforms.Compose([
253
+ torchvision.transforms.ToTensor(),
254
+ torchvision.transforms.Normalize(image_mean, image_std)
255
+ ])
256
+
257
+ # PIL images
258
+ # HD_transform pad images to size of multiiply of 336, 336
259
+ # convert to RGB first
260
+ images = [image.convert('RGB') for image in images]
261
+ elems = [HD_transform(im, hd_num = self.num_crops) for im in images]
262
+ # tensor transform and normalize
263
+ hd_images = [img_processor(im) for im in elems]
264
+ # create global image
265
+ global_image = [torch.nn.functional.interpolate(im.unsqueeze(0).float(), size=(336, 336), mode='bicubic',).to(im.dtype) for im in hd_images]
266
+
267
+ # [(3, h, w)], where h, w is multiple of 336
268
+ shapes = [[im.size(1), im.size(2)] for im in hd_images]
269
+ num_img_tokens = [int(((h//336)*(w//336)+1)*144 + 1 + (h//336+1)*12) for h, w in shapes]
270
+ # reshape to channel dimension -> (num_images, num_crops, 3, 336, 336)
271
+ # (1, 3, h//336, 336, w//336, 336) -> (1, h//336, w//336, 3, 336, 336) -> (h//336*w//336, 3, 336, 336)
272
+ hd_images_reshape = [im.reshape(1, 3, h//336, 336, w//336, 336).permute(0,2,4,1,3,5).reshape(-1, 3, 336, 336).contiguous() for im, (h, w) in zip(hd_images, shapes)]
273
+ # concat global image and local image
274
+ hd_images_reshape = [torch.cat([_global_image] + [_im], dim=0) for _global_image, _im in zip(global_image, hd_images_reshape)]
275
+
276
+ # pad to max_num_crops
277
+ image_transformed = [pad_to_max_num_crops_tensor(im, self.num_crops+1) for im in hd_images_reshape]
278
+ image_transformed = torch.stack(image_transformed, dim=0)
279
+ image_sizes = [torch.LongTensor(_shapes) for _shapes in shapes]
280
+ padded_images = image_transformed
281
+ image_sizes = shapes
282
+
283
+ data = {"pixel_values": padded_images,
284
+ "image_sizes": image_sizes,
285
+ "num_img_tokens": num_img_tokens
286
+ }
287
+
288
+ return BatchFeature(data=data, tensor_type=return_tensors)
289
+
290
+ AutoImageProcessor.register("Phi3VImageProcessor", Phi3VImageProcessor)
291
+
292
  transformers.Phi3VImageProcessor = Phi3VImageProcessor
293
 
294
  class Phi3VProcessor(ProcessorMixin):