Spaces:
Runtime error
Runtime error
File size: 5,595 Bytes
d1b8c9b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 |
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
from typing import List, Optional, Tuple
import torch
from data.retrieval_dataset import (
ImageToTextRetrievalDataset,
RetrievalTrainingDataset,
TextToImageRetrievalDataset,
)
from data.transforms import (
ALBEFTextTransform,
testing_image_transform,
training_image_transform,
)
from pytorch_lightning import LightningDataModule
from torch import Tensor
from torch.nn.utils.rnn import pad_sequence
from torch.utils.data import DataLoader, Dataset, DistributedSampler
class RetrievalDataModule(LightningDataModule):
"""
The Data Module for Retrieval task.
Args:
train_files (List[str]): The paths to training json files.
test_files (List[str]): The paths to testing json files.
image_root (str): The path to image data directory.
batch_size (int): The sampling batch size.
num_workers (int): The number of workers for the distributed mode.
"""
def __init__(
self,
train_files: List[str],
test_files: List[str],
image_root: str,
batch_size: int,
num_workers: int,
) -> None:
super().__init__()
self.train_dataset = RetrievalTrainingDataset(
train_files,
image_root,
training_image_transform(),
ALBEFTextTransform(truncate=True, max_seq_len=30, add_end_token=False),
)
self.image_dataset = ImageToTextRetrievalDataset(
test_files,
image_root,
testing_image_transform(),
)
self.text_dataset = TextToImageRetrievalDataset(
test_files,
ALBEFTextTransform(
truncate=True,
pad_to_max_seq_len=True,
max_seq_len=30,
add_end_token=False,
),
)
self.batch_size = batch_size
self.num_workers = num_workers
def _get_sampler(
self,
dataset: Dataset,
shuffle: bool,
is_distributed: bool,
num_tasks: int,
global_rank: int,
) -> Optional[DistributedSampler]:
# do not return a sampler if is not in distributed mode
# a default RandomSampler is used in this case
if not is_distributed:
return None
return DistributedSampler(
dataset, num_replicas=num_tasks, rank=global_rank, shuffle=shuffle
)
def train_dataloader(
self,
is_distributed: bool = False,
num_tasks: int = 0,
global_rank: int = 0,
drop_last: bool = True,
) -> DataLoader:
"""
DataLoader Outputs:
images (Tensor): Tensor of shape (B, C, W, H) of image inputs.
text (Tensor): Tensor of shape (B, L) of text inputs.
text_atts (Tensor): Tensor of shape (B, L) of text attention mask.
idx (Tensor): Tensor of shape (B) of image identifiers.
"""
sampler = self._get_sampler(
dataset=self.train_dataset,
shuffle=True,
is_distributed=is_distributed,
num_tasks=num_tasks,
global_rank=global_rank,
)
shuffle = sampler is None
return DataLoader(
self.train_dataset,
batch_size=self.batch_size,
num_workers=self.num_workers,
pin_memory=True,
sampler=sampler,
shuffle=shuffle,
collate_fn=retrieval_train_collate_fn,
drop_last=drop_last,
)
def image_dataloader(
self,
drop_last: bool = False,
) -> DataLoader:
"""
DataLoader Outputs:
images (Tensor): Tensor of shape (B, C, W, H) of image inputs.
"""
return DataLoader(
self.image_dataset,
batch_size=self.batch_size,
num_workers=self.num_workers,
pin_memory=True,
sampler=None,
shuffle=False,
collate_fn=None,
drop_last=drop_last,
)
def text_dataloader(
self,
drop_last: bool = False,
) -> DataLoader:
"""
DataLoader Outputs:
text (Tensor): Tensor of shape (B, L) of text inputs.
text_atts (Tensor): Tensor of shape (B, L) of text attention mask.
"""
return DataLoader(
self.text_dataset,
batch_size=self.batch_size,
num_workers=self.num_workers,
pin_memory=True,
sampler=None,
shuffle=False,
collate_fn=text_collate_fn,
drop_last=drop_last,
)
def retrieval_train_collate_fn(
batch: List[Tuple[Tensor, Tensor, int]]
) -> Tuple[Tensor, Tensor, Tensor, Tensor]:
image_list = []
text_list = []
idx_list = []
for image, text, idx in batch:
image_list.append(image)
text_list.append(text)
idx_list.append(idx)
images = torch.stack(image_list, dim=0)
text = pad_sequence(text_list, batch_first=True)
text_atts = (text != 0).type(torch.long)
idx = Tensor(idx_list).type(torch.long)
return (
images,
text,
text_atts,
idx,
)
def text_collate_fn(batch: List[Tensor]) -> Tuple[Tensor, Tensor]:
text = pad_sequence(batch, batch_first=True)
text_atts = (text != 0).type(torch.long)
return text, text_atts
|