Kaggle competition: Generative Dog Images
sub_df[["id", "toxic"]].to_csv("submission.csv", index=False )<count_duplicates>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
test_ids_df.duplicated("input_word_ids", keep=False ).sum()<count_duplicates>
batchSize = 48
imageSize = 64
nz = 128
real_label = 0.5
fake_label = 0
slope=0.2
val_ids_df.duplicated("input_word_ids", keep=False ).sum()<set_options>
print('Number of images: ', len(os.listdir('.. /input/all-dogs/all-dogs')) )
%matplotlib inline 
<set_options>
black_list=[]
%matplotlib inline 
<load_from_csv>
class FullCroppedDogsFolderDataset(torchvision.datasets.vision.VisionDataset):
def __init__(self, root, transform=None, target_transform=None):
super().__init__(root, transform=transform, target_transform=target_transform)
 self.transform = transform
self.target_transform = target_transform
self.samples = self._load_subfolders_images(self.root)
 if len(self.samples)== 0:
raise RuntimeError("Found 0 files in subfolders of: {}".format(self.root))
def _load_subfolders_images(self, root):
IMG_EXTENSIONS =('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
 def is_valid_file(x):
return torchvision.datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 required_transforms = torchvision.transforms.Compose([
torchvision.transforms.Resize(64),
torchvision.transforms.CenterCrop(64),
])
 imgs = []
paths = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames):
path = os.path.join(root, fname)
 paths.append(path)
 pbar = tqdm(paths, desc='Loading cropped images')
 for path in pbar:
file = path.split('/')[-1]
if is_valid_file(path)and file not in black_list:
img = torchvision.datasets.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(dirname for dirname in os.listdir('.. /input/annotation/Annotation/')if dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join('.. /input/annotation/Annotation', annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 max_dogs_per_image = 1
if len(objects)<= max_dogs_per_image:
for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = xmax - xmin
h = ymax - ymin
if h > w:
diff = h - w
xmin = xmin - diff/2
xmax = xmax + diff/2
xmax = min(xmax,img.width)
 xmin = max(xmin,0)
 if w > h:
diff = w - h
ymin = ymin - diff/2
ymax = ymax + diff/2
ymax = min(ymax,img.height)
 ymin = max(ymin,0)
 bbox =(xmin, ymin, xmax, ymax)
 object_img = required_transforms(img.crop(bbox))
imgs.append(object_img)
 pbar.set_postfix_str("{} cropped images loaded".format(len(imgs)))
 return imgs
def __getitem__(self, index):
sample = self.samples[index]
target = 1
if self.transform is not None:
sample = self.transform(sample)
 if self.target_transform is not None:
target = self.target_transform(target)
 return sample, target
def __len__(self):
return len(self.samples )
datadir = '.. /input'
gatrain = pd.read_csv(os.path.join(datadir,'gender_age_train.csv'),
index_col='device_id')
 gatest = pd.read_csv(os.path.join(datadir,'gender_age_test.csv'),
index_col = 'device_id')
 phone = pd.read_csv(os.path.join(datadir,'phone_brand_device_model.csv'))
phone = phone.drop_duplicates('device_id',keep='first' ).set_index('device_id')
 events = pd.read_csv(os.path.join(datadir,'events.csv'),
parse_dates=['timestamp'], index_col='event_id')
 appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),
usecols=['event_id','app_id','is_active'],
dtype={'is_active':bool})
 applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))<load_from_csv>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device )
datadir = '.. /input'
gatrain = pd.read_csv(os.path.join(datadir,'gender_age_train.csv'),
index_col='device_id')
 gatest = pd.read_csv(os.path.join(datadir,'gender_age_test.csv'),
index_col = 'device_id')
 phone = pd.read_csv(os.path.join(datadir,'phone_brand_device_model.csv'))
phone = phone.drop_duplicates('device_id',keep='first' ).set_index('device_id')
 events = pd.read_csv(os.path.join(datadir,'events.csv'),
parse_dates=['timestamp'], index_col='event_id')
 appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),
usecols=['event_id','app_id','is_active'],
dtype={'is_active':bool})
 applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))<feature_engineering>
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Conv')!= -1:
m.weight.data.normal_(0.0, 0.02)
 elif classname.find('BatchNorm')!= -1:
m.weight.data.normal_(1.0, 0.02)
 m.bias.data.fill_(0 )
gatrain['trainrow'] = np.arange(gatrain.shape[0])
 gatest['testrow'] = np.arange(gatest.shape[0] )<categorify>
class G(nn.Module):
def __init__(self, nz=128, channels=3):
super(G, self ).__init__()
self.nz = nz
self.channels = channels
self.main = nn.Sequential(
 nn.ConvTranspose2d(self.nz, 1024, 4, 1, 0, bias = False),
nn.BatchNorm2d(1024),
nn.ReLU(True),
nn.ConvTranspose2d(1024, 512, 4, 2, 1, bias = False),
nn.BatchNorm2d(512),
nn.ReLU(True),
nn.ConvTranspose2d(512, 256, 4, 2, 1, bias = False),
nn.BatchNorm2d(256),
nn.ReLU(True),
nn.ConvTranspose2d(256, 128, 4, 2, 1, bias = False),
nn.BatchNorm2d(128),
nn.ReLU(True),
nn.ConvTranspose2d(128, 64, 4, 2, 1, bias = False),
nn.BatchNorm2d(64),
nn.ReLU(True),
nn.ConvTranspose2d(64, self.channels, 3, 1, 1, bias = False),
nn.Tanh() 
)
 def forward(self, input):
output = self.main(input)
 return output
brandencoder = LabelEncoder().fit(phone.phone_brand)
 phone['brand'] = brandencoder.transform(phone['phone_brand'])
 gatrain['brand'] = phone['brand']
gatest['brand'] = phone['brand']
Xtr_brand = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.brand)))
 Xte_brand = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.brand)))
 print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))<categorify>
netG = G(nz ).to(device)
 netG.apply(weights_init )
brandencoder = LabelEncoder().fit(phone.phone_brand)
 phone['brand'] = brandencoder.transform(phone['phone_brand'])
 gatrain['brand'] = phone['brand']
gatest['brand'] = phone['brand']
Xtr_brand = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.brand)))
 Xte_brand = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.brand)))
 print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))<categorify>
class D(nn.Module):
def __init__(self, channels=3):
super(D, self ).__init__()
self.channels = channels
self.main = nn.Sequential(
 nn.Conv2d(self.channels, 64, 4, 2, 1, bias = False),
nn.LeakyReLU(slope, inplace = True),
nn.Conv2d(64, 128, 4, 2, 1, bias = False),
nn.BatchNorm2d(128),
nn.LeakyReLU(slope, inplace = True),
nn.Conv2d(128, 256, 4, 2, 1, bias = False),
nn.BatchNorm2d(256),
nn.LeakyReLU(slope, inplace = True),
nn.Conv2d(256, 512, 4, 2, 1, bias = False),
nn.BatchNorm2d(512),
nn.LeakyReLU(slope, inplace = True),
nn.Conv2d(512, 1, 4, 1, 0, bias = False),
nn.Sigmoid() 
)
 def forward(self, input):
output = self.main(input)
 return output.view(-1 )
m = phone.phone_brand.str.cat(phone.device_model)
 modelencoder = LabelEncoder().fit(m)
 phone['model'] = modelencoder.transform(m)
 gatrain['model'] = phone['model']
gatest['model'] = phone['model']
Xtr_model = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.model)))
 Xte_model = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.model)))
 print('Model features: train shape {}, test shape {}'.format(Xtr_model.shape, Xte_model.shape))<create_dataframe>
netD = D().to(device)
 netD.apply(weights_init )
appencoder = LabelEncoder().fit(appevents.app_id)
 appevents['app'] = appencoder.transform(appevents.app_id)
 napps = len(appencoder.classes_)
 deviceapps =(appevents.merge(events[['device_id']], how='left',left_on='event_id',right_index=True)
.groupby(['device_id','app'])['app'].agg(['size'])
.merge(gatrain[['trainrow']], how='left', left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left', left_index=True, right_index=True)
.reset_index())
 deviceapps.head()<prepare_x_and_y>
%%time
n_epochs = 286
criterion = nn.BCELoss()
optimizerD = optim.Adam(netD.parameters() , lr = 0.0003, betas =(0.5, 0.999))
optimizerG = optim.Adam(netG.parameters() , lr = 0.0003, betas =(0.5, 0.999))
lr_schedulerD = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizerD, factor=0.9, patience=37, min_lr=0.0002)
 lr_schedulerG = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizerG, factor=0.9, patience=37, min_lr=0.0002)
 G_losses = []
D_losses = []
for epoch in tqdm(range(n_epochs)) :
for i, data in enumerate(dataloader, 0):
D_losses_epoch = 0
G_losses_epoch = 0
netD.zero_grad()
input = data[0].to(device)
 batch_size = input.size(0)
 target = torch.full(( batch_size, 1), real_label, device=device)
 output = netD(input)
 errD_real = criterion(output, target)
 noise = Variable(torch.randn(batch_size, nz, 1, 1, device=device))
target = torch.full(( batch_size, 1), fake_label, device=device)
 fake = netG(noise)
 output = netD(fake.detach())
 errD_fake = criterion(output, target)
 errD = errD_real + errD_fake
errD.backward()
optimizerD.step()
D_losses_epoch += errD.item()
netG.zero_grad()
target = torch.full(( batch_size, 1), real_label, device=device)
 output = netD(fake)
 errG = criterion(output, target)
 errG.backward()
optimizerG.step()
G_losses_epoch += errG.item()
D_losses.append(errD.item())
 G_losses.append(errG.item())
 lr_schedulerD.step(D_losses_epoch)
 lr_schedulerG.step(G_losses_epoch)
 if epoch % 37 == 0:
for param_group in optimizerD.param_groups:
print('epoch', epoch, 'lr D:', param_group['lr'])
 for param_group in optimizerG.param_groups:
print('epoch', epoch, 'lr G:', param_group['lr'] )
d = deviceapps.dropna(subset=['trainrow'])
 Xtr_app = csr_matrix(( np.ones(d.shape[0]),(d.trainrow, d.app)) ,
shape=(gatrain.shape[0],napps))
d = deviceapps.dropna(subset=['testrow'])
 Xte_app = csr_matrix(( np.ones(d.shape[0]),(d.testrow, d.app)) ,
shape=(gatest.shape[0],napps))
print('Apps data: train shape {}, test shape {}'.format(Xtr_app.shape, Xte_app.shape))<categorify>
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values
<categorify><EOS> 
if not os.path.exists('.. /output_images'):
os.mkdir('.. /output_images')
 im_batch_size = 50
n_images=10000
for i_batch in range(0, n_images, im_batch_size):
z = truncated_normal(( im_batch_size, nz, 1, 1), threshold=1)
 gen_z = torch.from_numpy(z ).float().to(device)
 gen_images = netG(gen_z)
 gen_images.mul_(0.5 ).add_(0.5)
 images = gen_images.to('cpu' ).clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
 for i_image in range(gen_images.size(0)) :
save_image(gen_images[i_image, :, :, :], os.path.join('.. /output_images', f'image_{i_batch+i_image:05d}.png'))
shutil.make_archive('images', 'zip', '.. /output_images' )