Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<init_hyperparams>
%load_ext autoreload
%autoreload 2
%matplotlib inline
def G_arch(ch=64, attention='64', ksize='333333', dilation='111111'):
arch = {}
arch[512] = {'in_channels': [ch * item for item in [16, 16, 8, 8, 4, 2, 1]],
'out_channels': [ch * item for item in [16, 8, 8, 4, 2, 1, 1]],
'upsample': [True] * 7,
'resolution': [8, 16, 32, 64, 128, 256, 512],
'attention': {2 ** i:(2 ** i in [int(item)for item in attention.split('_')])
 for i in range(3, 10)}}
arch[256] = {'in_channels': [ch * item for item in [16, 16, 8, 8, 4, 2]],
'out_channels': [ch * item for item in [16, 8, 8, 4, 2, 1]],
'upsample': [True] * 6,
'resolution': [8, 16, 32, 64, 128, 256],
'attention': {2 ** i:(2 ** i in [int(item)for item in attention.split('_')])
 for i in range(3, 9)}}
arch[128] = {'in_channels': [ch * item for item in [16, 16, 8, 4, 2]],
'out_channels': [ch * item for item in [16, 8, 4, 2, 1]],
'upsample': [True] * 5,
'resolution': [8, 16, 32, 64, 128],
'attention': {2 ** i:(2 ** i in [int(item)for item in attention.split('_')])
 for i in range(3, 8)}}
arch[64] = {'in_channels': [ch * item for item in [16, 16, 8, 4]],
'out_channels': [ch * item for item in [16, 8, 4, 2]],
'upsample': [True] * 4,
'resolution': [8, 16, 32, 64],
'attention': {2 ** i:(2 ** i in [int(item)for item in attention.split('_')])
 for i in range(3, 7)}}
arch[32] = {'in_channels': [ch * item for item in [4, 4, 4]],
'out_channels': [ch * item for item in [4, 4, 4]],
'upsample': [True] * 3,
'resolution': [8, 16, 32],
'attention': {2 ** i:(2 ** i in [int(item)for item in attention.split('_')])
 for i in range(3, 6)}}
return arch
class Generator(nn.Module):
def __init__(self, G_ch=64, dim_z=128, bottom_width=4, resolution=128,
G_kernel_size=3, G_attn='64', n_classes=1000,
num_G_SVs=1, num_G_SV_itrs=1,
G_shared=True, shared_dim=0, hier=False,
cross_replica=False, mybn=False,
G_activation=nn.ReLU(inplace=False),
G_lr=5e-5, G_B1=0.0, G_B2=0.999, adam_eps=1e-8,
BN_eps=1e-5, SN_eps=1e-12, G_mixed_precision=False, G_fp16=False,
G_init='ortho', skip_init=False, no_optim=False,
G_param='SN', norm_style='bn',
**kwargs):
super(Generator, self ).__init__()
self.ch = G_ch
self.dim_z = dim_z
self.bottom_width = bottom_width
self.resolution = resolution
self.kernel_size = G_kernel_size
self.attention = G_attn
self.n_classes = n_classes
self.G_shared = G_shared
self.shared_dim = shared_dim if shared_dim > 0 else dim_z
self.hier = hier
self.cross_replica = cross_replica
self.mybn = mybn
self.activation = G_activation
self.init = G_init
self.G_param = G_param
self.norm_style = norm_style
self.BN_eps = BN_eps
self.SN_eps = SN_eps
self.fp16 = G_fp16
self.arch = G_arch(self.ch, self.attention)[resolution]
if self.hier:
self.num_slots = len(self.arch['in_channels'])+ 1
self.z_chunk_size =(self.dim_z // self.num_slots)
 self.dim_z = self.z_chunk_size * self.num_slots
else:
self.num_slots = 1
self.z_chunk_size = 0
if self.G_param == 'SN':
self.which_conv = partial(SNConv2d,
kernel_size=3, padding=1,
num_svs=num_G_SVs, num_itrs=num_G_SV_itrs,
eps=self.SN_eps)
 self.which_linear = partial(SNLinear,
num_svs=num_G_SVs, num_itrs=num_G_SV_itrs,
eps=self.SN_eps)
 else:
self.which_conv = partial(nn.Conv2d, kernel_size=3, padding=1)
 self.which_linear = nn.Linear
self.which_embedding = nn.Embedding
bn_linear =(partial(self.which_linear, bias=False)if self.G_shared else self.which_embedding)
 self.which_bn = partial(ccbn,
which_linear=bn_linear,
cross_replica=self.cross_replica,
mybn=self.mybn,
input_size=(self.shared_dim + self.z_chunk_size if self.G_shared else self.n_classes),
norm_style=self.norm_style,
eps=self.BN_eps)
 self.shared =(self.which_embedding(n_classes, self.shared_dim)if G_shared else identity())
 self.linear = self.which_linear(
 self.dim_z // self.num_slots,
self.arch['in_channels'][0] *(self.bottom_width ** 2))
self.blocks = []
for index in range(len(self.arch['out_channels'])) :
self.blocks += [[GBlock(
 in_channels=self.arch['in_channels'][index],
out_channels=self.arch['out_channels'][index],
which_conv=self.which_conv,
which_bn=self.which_bn,
activation=self.activation,
upsample=(partial(F.interpolate, scale_factor=2)if self.arch['upsample'][index] else None)
)]]
if self.arch['attention'][self.arch['resolution'][index]]:
print('Adding attention layer in G at resolution %d' % self.arch['resolution'][index])
 self.blocks[-1] += [Attention(self.arch['out_channels'][index], self.which_conv)]
self.blocks = nn.ModuleList([nn.ModuleList(block)for block in self.blocks])
 self.output_layer = nn.Sequential(
 bn(self.arch['out_channels'][-1], cross_replica=self.cross_replica, mybn=self.mybn),
self.activation,
self.which_conv(self.arch['out_channels'][-1], 3)
)
 if not skip_init:
self.init_weights()
if no_optim:
return
self.lr, self.B1, self.B2, self.adam_eps = G_lr, G_B1, G_B2, adam_eps
if G_mixed_precision:
print('Using fp16 adam in G...')
 self.optim = Adam16(params=self.parameters() , lr=self.lr,
betas=(self.B1, self.B2), weight_decay=0,
eps=self.adam_eps)
 else:
self.optim = optim.Adam(params=self.parameters() , lr=self.lr,
betas=(self.B1, self.B2), weight_decay=0,
eps=self.adam_eps)
 def init_weights(self):
self.param_count = 0
for module in self.modules() :
if(isinstance(module, nn.Conv2d)
 or isinstance(module, nn.Linear)
 or isinstance(module, nn.Embedding)) :
if self.init == 'ortho':
init.orthogonal_(module.weight)
 elif self.init == 'N02':
init.normal_(module.weight, 0, 0.02)
 elif self.init in ['glorot', 'xavier']:
init.xavier_uniform_(module.weight)
 else:
print('Init style not recognized...')
 self.param_count += sum([p.data.nelement() for p in module.parameters() ])
 print('Param count for G''s initialized parameters: %d' % self.param_count)
 def forward(self, z, y):
if self.hier:
zs = torch.split(z, self.z_chunk_size, 1)
 z = zs[0]
ys = [torch.cat([y, item], 1)for item in zs[1:]]
else:
ys = [y] * len(self.blocks)
 h = self.linear(z)
 h = h.view(h.size(0), -1, self.bottom_width, self.bottom_width)
 for index, blocklist in enumerate(self.blocks):
for block in blocklist:
h = block(h, ys[index])
 return torch.tanh(self.output_layer(h))
def D_arch(ch=64, attention='64', ksize='333333', dilation='111111'):
arch = {}
arch[256] = {'in_channels': [3] + [ch * item for item in [1, 2, 4, 8, 8, 16]],
'out_channels': [item * ch for item in [1, 2, 4, 8, 8, 16, 16]],
'downsample': [True] * 6 + [False],
'resolution': [128, 64, 32, 16, 8, 4, 4],
'attention': {2 ** i: 2 ** i in [int(item)for item in attention.split('_')]
for i in range(2, 8)}}
arch[128] = {'in_channels': [3] + [ch * item for item in [1, 2, 4, 8, 16]],
'out_channels': [item * ch for item in [1, 2, 4, 8, 16, 16]],
'downsample': [True] * 5 + [False],
'resolution': [64, 32, 16, 8, 4, 4],
'attention': {2 ** i: 2 ** i in [int(item)for item in attention.split('_')]
for i in range(2, 8)}}
arch[64] = {'in_channels': [3] + [ch * item for item in [1, 2, 4, 8]],
'out_channels': [item * ch for item in [1, 2, 4, 8, 16]],
'downsample': [True] * 4 + [False],
'resolution': [32, 16, 8, 4, 4],
'attention': {2 ** i: 2 ** i in [int(item)for item in attention.split('_')]
for i in range(2, 7)}}
arch[32] = {'in_channels': [3] + [item * ch for item in [4, 4, 4]],
'out_channels': [item * ch for item in [4, 4, 4, 4]],
'downsample': [True, True, False, False],
'resolution': [16, 16, 16, 16],
'attention': {2 ** i: 2 ** i in [int(item)for item in attention.split('_')]
for i in range(2, 6)}}
return arch
class Discriminator(nn.Module):
def __init__(self, D_ch=64, D_wide=True, resolution=128,
D_kernel_size=3, D_attn='64', n_classes=1000,
num_D_SVs=1, num_D_SV_itrs=1, D_activation=nn.ReLU(inplace=False),
D_lr=2e-4, D_B1=0.0, D_B2=0.999, adam_eps=1e-8,
SN_eps=1e-12, output_dim=1, D_mixed_precision=False, D_fp16=False,
D_init='ortho', skip_init=False, D_param='SN', **kwargs):
super(Discriminator, self ).__init__()
self.ch = D_ch
self.D_wide = D_wide
self.resolution = resolution
self.kernel_size = D_kernel_size
self.attention = D_attn
self.n_classes = n_classes
self.activation = D_activation
self.init = D_init
self.D_param = D_param
self.SN_eps = SN_eps
self.fp16 = D_fp16
self.arch = D_arch(self.ch, self.attention)[resolution]
if self.D_param == 'SN':
self.which_conv = partial(SNConv2d,
kernel_size=3, padding=1,
num_svs=num_D_SVs, num_itrs=num_D_SV_itrs,
eps=self.SN_eps)
 self.which_linear = partial(SNLinear,
num_svs=num_D_SVs, num_itrs=num_D_SV_itrs,
eps=self.SN_eps)
 self.which_embedding = partial(SNEmbedding,
num_svs=num_D_SVs, num_itrs=num_D_SV_itrs,
eps=self.SN_eps)
 self.blocks = []
for index in range(len(self.arch['out_channels'])) :
self.blocks += [[DBlock(
 in_channels=self.arch['in_channels'][index],
out_channels=self.arch['out_channels'][index],
which_conv=self.which_conv,
wide=self.D_wide,
activation=self.activation,
preactivation=(index > 0),
downsample=(nn.AvgPool2d(2)if self.arch['downsample'][index] else None)
)]]
if self.arch['attention'][self.arch['resolution'][index]]:
print('Adding attention layer in D at resolution %d' % self.arch['resolution'][index])
 self.blocks[-1] += [Attention(self.arch['out_channels'][index], self.which_conv)]
self.blocks = nn.ModuleList([nn.ModuleList(block)for block in self.blocks])
 self.linear = self.which_linear(self.arch['out_channels'][-1], output_dim)
 self.embed = self.which_embedding(self.n_classes, self.arch['out_channels'][-1])
 if not skip_init:
self.init_weights()
self.lr, self.B1, self.B2, self.adam_eps = D_lr, D_B1, D_B2, adam_eps
if D_mixed_precision:
print('Using fp16 adam in D...')
 self.optim = Adam16(params=self.parameters() , lr=self.lr,
betas=(self.B1, self.B2), weight_decay=0, eps=self.adam_eps)
 else:
self.optim = optim.Adam(params=self.parameters() , lr=self.lr,
betas=(self.B1, self.B2), weight_decay=0, eps=self.adam_eps)
 def init_weights(self):
self.param_count = 0
for module in self.modules() :
if(isinstance(module, nn.Conv2d)
 or isinstance(module, nn.Linear)
 or isinstance(module, nn.Embedding)) :
if self.init == 'ortho':
init.orthogonal_(module.weight)
 elif self.init == 'N02':
init.normal_(module.weight, 0, 0.02)
 elif self.init in ['glorot', 'xavier']:
init.xavier_uniform_(module.weight)
 else:
print('Init style not recognized...')
 self.param_count += sum([p.data.nelement() for p in module.parameters() ])
 print('Param count for D''s initialized parameters: %d' % self.param_count)
 def forward(self, x, y=None):
h = x
for index, blocklist in enumerate(self.blocks):
for block in blocklist:
h = block(h)
 h = torch.sum(self.activation(h), [2, 3])
 out = self.linear(h)
 out = out + torch.sum(self.embed(y)* h, 1, keepdim=True)
 return out
class G_D(nn.Module):
def __init__(self, G, D):
super(G_D, self ).__init__()
self.G = G
self.D = D
def forward(self, z, gy, x=None, dy=None, train_G=False, return_G_z=False,
split_D=False):
with torch.set_grad_enabled(train_G):
G_z = self.G(z, self.G.shared(gy))
if self.G.fp16 and not self.D.fp16:
G_z = G_z.float()
if self.D.fp16 and not self.G.fp16:
G_z = G_z.half()
if split_D:
D_fake = self.D(G_z, gy)
 if x is not None:
D_real = self.D(x, dy)
 return D_fake, D_real
else:
if return_G_z:
return D_fake, G_z
else:
return D_fake
else:
D_input = torch.cat([G_z, x], 0)if x is not None else G_z
D_class = torch.cat([gy, dy], 0)if dy is not None else gy
D_out = self.D(D_input, D_class)
 if x is not None:
return torch.split(D_out, [G_z.shape[0], x.shape[0]])
 else:
if return_G_z:
return D_out, G_z
else:
return D_out<normalization>
from fastai.tabular import *
from sklearn.model_selection import train_test_split, StratifiedKFold, KFold, cross_val_score, GridSearchCV, RepeatedStratifiedKFold
import time
import lightgbm as lgb
import xgboost as xgb
import ast
from sklearn.metrics import mean_squared_error
import seaborn as sns
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.decomposition import TruncatedSVD
class MyImg() :
def __init__(self, img, tfm):
self.px = np.array(img)
 self.tfm = tfm
@property
def size(self):
h, w, _ = self.px.shape
return min(w, h)
 def pad(img, padding_mode='reflect'):
p = math.ceil(( max(img.size)- min(img.size)) / 2)
 p_horr = p if img.width < img.height else 0
p_vert = p if img.height < img.width else 0
img = T.Pad(( p_horr, p_vert), padding_mode=padding_mode )(img)
 if img.width != img.height:
s = min(img.size)
 img = img.crop(( 0, 0, s, s))
return img
def take_top(img):
size = min(img.size)
 bbox =(0, 0, size, size)
 return img.crop(bbox)
 def take_diagonal(img):
w, h = img.size
size = min(w, h)
 bbox_l =(0, 0, size, size)
 bbox_r =(w - size, h - size, w, h)
 return [img.crop(bbox_l), img.crop(bbox_r)]
resize = T.Resize(IMG_SIZE, interpolation=Image.LANCZOS)
 resize2x = T.Resize(IMG_SIZE_2, interpolation=Image.LANCZOS)
 center_crop = T.Compose([resize, T.CenterCrop(IMG_SIZE)])
 center_crop2x = T.Compose([resize2x, T.CenterCrop(IMG_SIZE_2)])
 top_crop = T.Compose([T.Lambda(take_top), resize])
 top_crop2x = T.Compose([T.Lambda(take_top), resize2x])
 two_crops = T.Compose([
resize, T.FiveCrop(IMG_SIZE),
T.Lambda(lambda imgs: [imgs[i] for i in [0, 2]])
 ])
 two_crops2x = T.Compose([resize2x, T.Lambda(take_diagonal)])
 pad_only = T.Compose([T.Lambda(pad), resize])
 pad_only2x = T.Compose([T.Lambda(pad), resize2x])
 @jit(nopython=True)
 def pad_one_dim(clow, chigh, pad, cmax):
clow = max(0, clow - pad)
 chigh = min(cmax, chigh + pad)
 return clow, chigh, chigh - clow
def calc_bbox(obj, img_w, img_h, zoom=0.0, try_square=True):
bndbox = obj.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 xmin, xmax, obj_w = pad_one_dim(xmin, xmax, 0, img_w)
 ymin, ymax, obj_h = pad_one_dim(ymin, ymax, 0, img_h)
 if zoom != 0.0:
pad_w = obj_w * zoom / 2
pad_h = obj_h * zoom / 2
xmin, xmax, obj_w = pad_one_dim(xmin, xmax, pad_w, img_w)
 ymin, ymax, obj_h = pad_one_dim(ymin, ymax, pad_h, img_h)
 if try_square:
if obj_w > obj_h:
pad =(obj_w - obj_h)/ 2
ymin, ymax, obj_h = pad_one_dim(ymin, ymax, pad, img_h)
 elif obj_h > obj_w:
pad =(obj_h - obj_w)/ 2
xmin, xmax, obj_w = pad_one_dim(xmin, xmax, pad, img_w)
 if obj_w > obj_h:
pad = obj_w - obj_h
ymin, ymax, obj_h = pad_one_dim(ymin, ymax, pad, img_h)
 elif obj_h > obj_w:
pad = obj_h - obj_w
xmin, xmax, obj_w = pad_one_dim(xmin, xmax, pad, img_w)
 return int(xmin), int(ymin), int(xmax), int(ymax)
 @jit(nopython=True)
 def bb2wh(bbox):
width = bbox[2] - bbox[0]
height = bbox[3] - bbox[1]
return width, height
def make_x2res(img, bbox):
if min(bb2wh(bbox)) < IMG_SIZE_2: return
ar = img.width / img.height
if ar == 1.0:
tfm_img = resize2x(img)
 elif 1.0 < ar < 1.15:
tfm_img = center_crop2x(img)
 elif 1.15 < ar < 1.25:
tfm_img = pad_only2x(img)
 elif 1.25 < ar < 1.5:
tfm_img = two_crops2x(img)
 elif 1.0 < 1/ar < 1.6:
tfm_img = top_crop2x(img)
 else:
tfm_img = None
return tfm_img
def add_sample(samples, label, tfm, imgs, labels):
if not samples: return
elif isinstance(samples, Image.Image):
imgs.append(MyImg(samples, tfm))
labels.append(label)
 elif isinstance(samples, list):
imgs.extend([MyImg(s, tfm)for s in samples])
 labels.extend([label] * len(samples))
else: assert False
def is_valid_file(x):
return datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 class DogsDataSet(datasets.vision.VisionDataset):
def __init__(self, root, transforms, target_transform=None, max_samples=None):
super().__init__(root, transform=None)
 assert isinstance(transforms, list)and len(transforms)== 3
self.transforms = transforms
self.target_transform = target_transform
self.max_samples = max_samples
self.classes = {}
imgs, labels = self._load_subfolders_images(self.root)
 assert len(imgs)== len(labels)
 if len(imgs)== 0:
raise RuntimeError(f'Found 0 files in subfolders of: {self.root}')
 self.imgs = imgs
self.labels = labels
def _create_or_get_class(self, name):
try:
label = self.classes[name]
except KeyError:
label = len(self.classes)
 self.classes[name] = label
return label
def _load_subfolders_images(self, root):
NO_ZOOM, LIGHT_ZOOM, MEDIUM_ZOOM = 0.0, 0.08, 0.12
n_pad, n_center, n_top, n_2crops, n_skip, n_dup, n_noop = 0, 0, 0, 0, 0, 0, 0
imgs, labels, paths = [], [], []
add_sample_ = partial(add_sample, imgs=imgs, labels=labels)
 for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames):
path = os.path.join(root, fname)
 paths.append(path)
 if self.max_samples:
paths = paths[:self.max_samples]
for path in paths:
if not is_valid_file(path): continue
img = datasets.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(dirname for dirname in os.listdir(ANNOT_PATH)if dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join(ANNOT_PATH, annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
name = o.find('name' ).text
label = self._create_or_get_class(name)
 prev_bbox, tfm_imgs = None, None
bbox = calc_bbox(o, img_w=img.width, img_h=img.height, zoom=LIGHT_ZOOM)
 obj_img = img.crop(bbox)
 add_sample_(make_x2res(obj_img, bbox), label, 2)
 bbox = calc_bbox(o, img_w=img.width, img_h=img.height)
 if min(bb2wh(bbox)) < IMG_SIZE:
n_skip += 1; continue
obj_img = img.crop(bbox)
 ar = obj_img.width / obj_img.height
if ar == 1.0:
tfm_imgs = [resize(obj_img)]; n_noop += 1
elif 1.0 < ar < 1.3:
tfm_imgs = [center_crop(obj_img), pad_only(obj_img)]
n_center += 1; n_pad += 1
elif 1.3 <= ar < 1.5:
tfm_imgs = two_crops(obj_img)+ [pad_only(obj_img)]
n_2crops += 2; n_pad += 1
elif 1.5 <= ar < 2.0:
tfm_imgs = two_crops(obj_img); n_2crops += 2
elif 1.0 < 1/ar < 1.5:
tfm_imgs = [top_crop(obj_img), pad_only(obj_img)]
n_top += 1; n_pad += 1
elif 1.5 <= 1/ar < 1.8:
tfm_imgs = [top_crop(obj_img)]; n_top += 1
else:
tfm_imgs = None; n_skip += 1
add_sample_(tfm_imgs, label, 0)
 add_sample_(make_x2res(obj_img, bbox), label, 1)
 prev_bbox = bbox
bbox = calc_bbox(o, img_w=img.width, img_h=img.height, zoom=MEDIUM_ZOOM, try_square=False)
 if bbox == prev_bbox:
n_dup += 1; continue
if min(bb2wh(bbox)) < IMG_SIZE_2: continue
obj_img = img.crop(bbox)
 ar = obj_img.width / obj_img.height
if 1.3 < ar < 1.5:
tfm_imgs = two_crops(obj_img); n_2crops += 2
elif 1.05 < 1/ar < 1.6:
tfm_imgs = top_crop(obj_img); n_top += 1
else: continue
add_sample_(tfm_imgs, label, 0)
 add_sample_(make_x2res(obj_img, bbox), label, 1)
 prev_bbox = bbox
n_x1, n_x2 = 0, 0
for i, img in enumerate(imgs):
if img.size == IMG_SIZE:
n_x1 +=1
else:
n_x2 +=1
print(f'Found {len(self.classes)} classes
Loaded 64x64 {n_x1} images
'
f'Loaded 128x128 {n_x2} images
')
 print(f'Pad only: {n_pad}
Crop center: {n_center}
'
f'Crop top: {n_top}
Crop 2 times: {n_2crops}
'
f'Take as-is: {n_noop}
Skip: {n_skip}
Same bbox: {n_dup}')
 return imgs, labels
def __getitem__(self, index):
img = self.imgs[index]
label = self.labels[index]
tfms = self.transforms[img.tfm]
img = tfms(image=img.px)['image']
if self.target_transform:
label = self.target_transform(label)
 return img, label
def __len__(self):
return len(self.imgs)
 def create_runtime_tfms() :
mean, std = [0.5]*3, [0.5]*3
resize_to_64 = A.SmallestMaxSize(IMG_SIZE, interpolation=cv2.INTER_LANCZOS4)
 out = [A.HorizontalFlip(p=0.5), A.Normalize(mean=mean, std=std), ToTensor() ]
rand_crop = A.Compose([
A.SmallestMaxSize(IMG_SIZE + 8, interpolation=cv2.INTER_LANCZOS4),
A.RandomCrop(IMG_SIZE, IMG_SIZE)
 ])
 affine_1 = A.ShiftScaleRotate(
 shift_limit=0, scale_limit=0.1, rotate_limit=8,
interpolation=cv2.INTER_CUBIC,
border_mode=cv2.BORDER_REFLECT_101, p=1.0)
 affine_1 = A.Compose([affine_1, resize_to_64])
 affine_2 = A.ShiftScaleRotate(
 shift_limit=0.06, scale_limit=(-0.06, 0.18), rotate_limit=6,
interpolation=cv2.INTER_CUBIC,
border_mode=cv2.BORDER_REFLECT_101, p=1.0)
 affine_2 = A.Compose([affine_2, resize_to_64])
 tfm_0 = A.Compose(out)
 tfm_1 = A.Compose([A.OneOrOther(affine_1, rand_crop, p=1.0), *out])
 tfm_2 = A.Compose([affine_2, *out])
 return [tfm_0, tfm_1, tfm_2]
def get_data_loaders(data_root=None, batch_size=32, num_workers=2, shuffle=True,
pin_memory=True, drop_last=True):
print('Using dataset root location %s' % data_root)
 train_set = DogsDataSet(data_root, create_runtime_tfms())
 loaders = []
loader_kwargs = {'num_workers': num_workers, 'pin_memory': pin_memory,
'drop_last': drop_last}
train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=shuffle, **loader_kwargs)
 loaders.append(train_loader)
 return loaders<categorify>
PATH = ".. /input/tmdb-box-office-prediction/"
def seed_rng(seed):
torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 np.random.seed(seed)
 def update_config_roots(config):
if config['base_root']:
print('Pegging all root folders to base root %s' % config['base_root'])
 for key in ['data', 'weights', 'logs', 'samples']:
config['%s_root' % key] = '%s/%s' %(config['base_root'], key)
 return config
def prepare_root(config):
for key in ['weights_root', 'logs_root', 'samples_root']:
if not os.path.exists(config[key]):
print('Making directory %s for %s...' %(config[key], key))
os.mkdir(config[key])
 def join_strings(base_string, strings):
return base_string.join([item for item in strings if item] )<prepare_x_and_y>
train = pd.read_csv(f'{PATH}train.csv', parse_dates=['release_date'])
 test = pd.read_csv(f'{PATH}test.csv', parse_dates=['release_date'] )
def run(G, D, GD, loaders, state_dict, config, device, G_ema=None, ema=None):
G_batch_size = max(config['G_batch_size'], config['batch_size'])
 z_, y_ = prepare_z_y(
 G_batch_size, G.dim_z, config['n_classes'], device=device, fp16=config['G_fp16'])
 fixed_z, fixed_y = prepare_z_y(
 G_batch_size, G.dim_z, config['n_classes'], device=device, fp16=config['G_fp16'])
 fixed_z.sample_()
fixed_y.sample_()
train = create_train_fn(G, D, GD, z_, y_, ema, state_dict, config)
 print('Beginning training at epoch %d...' % state_dict['epoch'])
 start_time = time.perf_counter()
total_iters = config['num_epochs'] * len(loaders[0])
 for epoch in range(state_dict['epoch'], config['num_epochs']):
for i,(x, y)in enumerate(loaders[0]):
state_dict['itr'] += 1
G.train()
D.train()
if config['ema']:
G_ema.train()
if config['D_fp16']:
x, y = x.to(device ).half() , y.to(device)
 else:
x, y = x.to(device), y.to(device)
 metrics = train(x, y)
 if not(state_dict['itr'] % config['log_interval']):
curr_time = time.perf_counter()
curr_time_str = datetime.datetime.fromtimestamp(curr_time ).strftime('%H:%M:%S')
 elapsed = str(datetime.timedelta(seconds=(curr_time - start_time)))
 log =(
 "[{}] [{}] [{} / {}] Ep {}, ".format(curr_time_str, elapsed, state_dict['itr'], total_iters, epoch)+
', '.join(['%s : %+4.3f' %(key, metrics[key])for key in metrics])
)
 print(log)
 if not(state_dict['itr'] % config['save_every']):
if config['G_eval_mode']:
print('Switchin G to eval mode...')
 G.eval()
save_and_sample(G, G_ema, fixed_z, fixed_y, config)
 if config['stop_after'] > 0 and int(time.perf_counter() - start_time)> config['stop_after']:
print("Time limit reached! Stopping training!")
 return
state_dict['epoch'] += 1<split>
train_votes = pd.read_csv('.. /input/tmdb-prediction-votes/trainRatingTotalVotes.csv')
 test_votes = pd.read_csv('.. /input/tmdb-prediction-votes/testRatingTotalVotes.csv')
 train = pd.merge(train, train_votes, how='left', on=['imdb_id'])
 test = pd.merge(test, test_votes, how='left', on=['imdb_id'] )
def trunc_trick(bs, z_dim, bound=2.0):
z = torch.randn(bs, z_dim)
 while z.min() < -bound or bound < z.max() :
z = z.where(( -bound < z)&(z < bound), torch.randn_like(z))
return z
def collect_bn_stats(G, n_samples, config):
im_batch_size = config['n_classes']
G.train()
for i_batch in range(0, n_samples, im_batch_size):
with torch.no_grad() :
z = torch.randn(im_batch_size, G.dim_z, device=device)
 y = torch.arange(im_batch_size ).to(device)
 images = G(z, G.shared(y)).float().cpu()
def generate_images(out_dir, G, n_images, config):
im_batch_size = config['n_classes']
z_bound = config['trunc_z']
if z_bound > 0.0: print(f'Truncating z to(-{z_bound}, {z_bound})')
 for i_batch in range(0, n_images, im_batch_size):
with torch.no_grad() :
if z_bound > 0.0:
z = trunc_trick(im_batch_size, G.dim_z, bound=z_bound ).to(device)
 else:
z = torch.randn(im_batch_size, G.dim_z, device=device)
 y = torch.arange(im_batch_size ).to(device)
 images = G(z, G.shared(y)).float().cpu()
if i_batch + im_batch_size > n_images:
n_last_images = n_images - i_batch
print(f'Taking only {n_last_images} images from the last batch...')
 images = images[:n_last_images]
for i_image, image in enumerate(images):
fname = os.path.join(out_dir, f'image_{i_batch+i_image:05d}.png')
 image = denorm(image)
 if config['denoise']:
image = image * 256
image = image.numpy().transpose(1, 2, 0 ).astype('uint8')
 image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
 image = cv2.fastNlMeansDenoisingColored(
 src=image, dst=None,
h=config['denoise_str_lum'],
hColor=config['denoise_str_col'],
templateWindowSize=config['denoise_kernel_size'],
searchWindowSize=config['denoise_search_window'] 
)
 cv2.imwrite(fname, image)
 else:
torchvision.utils.save_image(image, fname)
 def show_saved_samples(n_images=256, cols=8, size=3):
rows = math.ceil(n_images / cols)
 _, axes = plt.subplots(rows, cols, figsize=(cols * size, rows * size))
for i, ax in enumerate(axes.flatten()):
img = Image.open(f'{OUT_DIR}/image_{i:05d}.png')
 ax.get_xaxis().set_visible(False)
 ax.get_yaxis().set_visible(False)
 ax.imshow(img)
 plt.subplots_adjust(wspace=0.0, hspace=0.0)
 plt.show()<choose_model_class>
dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
train = text_to_dict(train)
 test = text_to_dict(test )
activation_dict = {
'inplace_relu': nn.ReLU(inplace=True),
'relu': nn.ReLU(inplace=False),
'ir': nn.ReLU(inplace=True),
}
config = {
'num_workers': 2,
'pin_memory': True,
'shuffle': True,
'use_multiepoch_sampler': False,
'model': 'BigGAN',
'G_param': 'SN',
'D_param': 'SN',
'G_ch': 64,
'D_ch': 64,
'G_depth': 1,
'D_depth': 1,
'D_wide': True,
'G_shared': False,
'shared_dim': 0,
'dim_z': 128,
'z_var': 1.0,
'hier': False,
'cross_replica': False,
'mybn': False,
'G_nl': 'relu',
'D_nl': 'relu',
'G_attn': '64',
'D_attn': '64',
'norm_style': 'bn',
'seed': 0,
'G_init': 'ortho',
'D_init': 'ortho',
'skip_init': False,
'G_lr': 5e-5,
'D_lr': 2e-4,
'G_B1': 0.0,
'D_B1': 0.0,
'G_B2': 0.999,
'D_B2': 0.999,
'batch_size': 64,
'G_batch_size': 0,
'num_G_accumulations': 1,
'num_D_steps': 2,
'num_D_accumulations': 1,
'split_D': False,
'num_epochs': 100,
'parallel': False,
'G_fp16': False,
'D_fp16': False,
'D_mixed_precision': False,
'G_mixed_precision': False,
'accumulate_stats': False,
'num_standing_accumulations': 16,
'G_eval_mode': False,
'save_every': 2000,
'num_save_copies': 2,
'num_best_copies': 2,
'which_best': 'IS',
'no_fid': False,
'test_every': 5000,
'num_inception_images': 50000,
'hashname': False,
'base_root': '',
'data_root': '.. /data',
'weights_root': '.. /weights',
'logs_root': '.. /logs',
'samples_root': '.. /samples',
'pbar': 'mine',
'name_suffix': '',
'experiment_name': '',
'config_from_name': False,
'ema': False,
'ema_decay': 0.9999,
'use_ema': False,
'ema_start': 0,
'adam_eps': 1e-8,
'BN_eps': 1e-5,
'SN_eps': 1e-8,
'num_G_SVs': 1,
'num_D_SVs': 1,
'num_G_SV_itrs': 1,
'num_D_SV_itrs': 1,
'G_ortho': 0.0,
'D_ortho': 0.0,
'toggle_grads': True,
'which_train_fn': 'GAN',
'load_weights': '',
'resume': False,
'logstyle': '%3.3e',
'log_G_spectra': False,
'log_D_spectra': False,
'sv_log_interval': 10,
'log_interval': 100,
}
config['resolution'] = 64
config['n_classes'] = 120
config['data_root'] = INPUT_PATH<set_options>
def build_category_list(x, field, feature):
regex = re.compile('[^0-9a-zA-Z_]')
 category_list = ""
for d in x:
new_category = regex.sub('', d[field].lower().replace(" ","_"))
category_list += new_category + ","
return category_list.strip().strip("," ).split(",")
 target_fields = {'belongs_to_collection': 'name', 'genres': 'name',
'production_countries': 'iso_3166_1', 'production_companies': 'name',
'spoken_languages': 'iso_639_1', 'Keywords': 'name', 'cast': 'name'
}
for k,v in target_fields.items() :
train[k] = train[k].apply(lambda x: build_category_list(x, v, k))
test[k] = test[k].apply(lambda x: build_category_list(x, v, k))
target_fields = {'cast':{'field':'name', 'role_field':'order', 'role_values':[0,1,2]}}
config['num_epochs'] = 100
config['num_D_steps'] = 1
config['G_lr'] = 1e-4 * 0.85
config['D_lr'] = 4e-4 * 0.85
config['D_B2'] = 0.999
config['G_B2'] = 0.999
config['G_ch'] = 32
config['D_ch'] = 64
config['G_attn'] = '32'
config['D_attn'] = '32'
config['G_nl'] = 'inplace_relu'
config['D_nl'] = 'inplace_relu'
config['G_shared'] = True
config['shared_dim'] = 128
config['dim_z'] = 120
config['hier'] = True
config['SN_eps'] = 1e-8
config['BN_eps'] = 1e-5
config['adam_eps'] = 1e-8
config['G_ortho'] = 0.0
config['G_init'] = 'ortho'
config['D_init'] = 'ortho'
config['G_eval_mode'] = True
config['save_every'] = 5000
config['log_interval'] = 500
config['batch_size'] = 32
config['ema'] = True
config['use_ema'] = True
config['ema_decay'] = 0.9999
config['ema_start'] = 50_000
config['stop_after'] = 31_500
config['trunc_z'] = 0.0
config['denoise'] = True
config['denoise_str_lum'] = 2.65
config['denoise_str_col'] = 1.0
config['denoise_kernel_size'] = 7
config['denoise_search_window'] = 11<set_options>
train.loc[train['id'] == 16,'revenue'] = 192864
train.loc[train['id'] == 90,'budget'] = 30000000
train.loc[train['id'] == 118,'budget'] = 60000000
train.loc[train['id'] == 149,'budget'] = 18000000
train.loc[train['id'] == 313,'revenue'] = 12000000
train.loc[train['id'] == 451,'revenue'] = 12000000
train.loc[train['id'] == 464,'budget'] = 20000000
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 930000
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 819,'budget'] = 90000000
train.loc[train['id'] == 850,'budget'] = 90000000
train.loc[train['id'] == 1112,'budget'] = 7500000
train.loc[train['id'] == 1131,'budget'] = 4300000
train.loc[train['id'] == 1359,'budget'] = 10000000
train.loc[train['id'] == 1542,'budget'] = 1
train.loc[train['id'] == 1542,'budget'] = 15800000
train.loc[train['id'] == 1571,'budget'] = 4000000
train.loc[train['id'] == 1714,'budget'] = 46000000
train.loc[train['id'] == 1721,'budget'] = 17500000
train.loc[train['id'] == 1865,'revenue'] = 25000000
train.loc[train['id'] == 2268,'budget'] = 17500000
train.loc[train['id'] == 2491,'revenue'] = 6800000
train.loc[train['id'] == 2602,'budget'] = 31000000
train.loc[train['id'] == 2612,'budget'] = 15000000
train.loc[train['id'] == 2696,'budget'] = 10000000
train.loc[train['id'] == 2801,'budget'] = 10000000
config = update_config_roots(config)
 device = 'cuda'
seed_rng(1234)
 prepare_root(config)
 torch.backends.cudnn.benchmark = True<define_variables>
test.loc[test['id'] == 3889,'budget'] = 15000000
test.loc[test['id'] == 6733,'budget'] = 5000000
test.loc[test['id'] == 3197,'budget'] = 8000000
test.loc[test['id'] == 6683,'budget'] = 50000000
test.loc[test['id'] == 5704,'budget'] = 4300000
test.loc[test['id'] == 6109,'budget'] = 281756
test.loc[test['id'] == 7242,'budget'] = 10000000
test.loc[test['id'] == 7021,'budget'] = 17540562
test.loc[test['id'] == 5591,'budget'] = 4000000
test.loc[test['id'] == 4282,'budget'] = 20000000
%%time
D_batch_size =(config['batch_size'] * config['num_D_steps'] * config['num_D_accumulations'])
 loaders = get_data_loaders(
 data_root=INPUT_PATH,
batch_size=D_batch_size,
num_workers=config['num_workers'],
shuffle=config['shuffle'],
pin_memory=config['pin_memory'],
drop_last=True
)<define_search_model>
power_six = train.id[train.budget > 1000][train.revenue < 100]
for k in power_six :
train.loc[train['id'] == k,'revenue'] = train.loc[train['id'] == k,'revenue'] * 1000000
config['G_activation'] = activation_dict[config['G_nl']]
config['D_activation'] = activation_dict[config['D_nl']]
G = Generator(**config ).to(device)
 D = Discriminator(**config ).to(device)
 if config['ema']:
print('Preparing EMA for G with decay of {}'.format(config['ema_decay']))
G_ema = Generator(**{**config, 'skip_init': True, 'no_optim': True} ).to(device)
 ema = apply_ema(G, G_ema, config['ema_decay'], config['ema_start'])
 else:
G_ema, ema = None, None
GD = G_D(G, D)
 print('Number of params in G: {} D: {}'.format(
 *[sum([p.data.nelement() for p in net.parameters() ])for net in [G, D]]))<define_variables>
def multi_hot_encode(df, column_name, mlb=MultiLabelBinarizer()):
encoded = pd.DataFrame(mlb.fit_transform(df[column_name]))
encoded.columns = [f'{column_name}_{i}'.format(i)for i in mlb.classes_]
return mlb, encoded
state_dict = {
'itr': 0,
'epoch': 0,
'config': config
}<train_model>
def conv_column_to_SVD(df, column_name, size=10, sparse_mlb=MultiLabelBinarizer(sparse_output=True)) :
sparse_matrix = sparse_mlb.fit_transform(df[column_name])
 sparse_SVD = TruncatedSVD(size)
 sparse_TSVD = pd.DataFrame(sparse_SVD.fit_transform(sparse_matrix))
sparse_TSVD.columns = [f'{column_name}_{i}' for i in range(size)]
return sparse_mlb, sparse_TSVD
run(G, D, GD, loaders, state_dict, config, device, G_ema=G_ema, ema=ema )<set_options>
genre_mlb, genre_encoded = multi_hot_encode(train,'genres')
 cast_sparse_mlb, cast_TSVD = conv_column_to_SVD(train, 'cast', 10)
 keywords_sparse_mlb, keywords_TSVD = conv_column_to_SVD(train, 'Keywords', 10)
 languages_mlb, languages_TSVD = conv_column_to_SVD(train, 'spoken_languages', 5)
 prodcomp_mlb, prodcomp_TSVD = conv_column_to_SVD(train, 'production_companies', 5)
 prodcountries_mlb, prodcountries_TSVD = conv_column_to_SVD(train, 'production_countries', 3 )
%%time
if not os.path.exists(OUT_DIR):
os.mkdir(OUT_DIR)
 which_G = G_ema if config['ema'] and config['use_ema'] else G
if config['use_ema']:
collect_bn_stats(G_ema, 20_000, config)
 gc.collect()
generate_images(OUT_DIR, which_G, 10_000, config)
 shutil.make_archive('images', 'zip', OUT_DIR )<set_options>
_,test_genre= multi_hot_encode(test,'genres',genre_mlb)
 _,test_cast = conv_column_to_SVD(test, 'cast', 10)
 _,test_keywords = conv_column_to_SVD(test, 'Keywords', 10)
 _,test_languages = conv_column_to_SVD(test, 'spoken_languages', 5)
 _,test_prodcomp = conv_column_to_SVD(test, 'production_companies', 5)
 _,test_prodcountries = conv_column_to_SVD(test, 'production_countries', 3 )
%load_ext autoreload
%autoreload 2
%matplotlib inline
gc.enable()
torch.__version__<define_variables>
train = train.join(genre_encoded)
 train = train.join(cast_TSVD)
 train = train.join(keywords_TSVD)
 train = train.join(languages_TSVD)
 train = train.join(prodcomp_TSVD)
 train = train.join(prodcountries_TSVD )
kernel_mode = True
if kernel_mode:
show_metric = False
show_animation = False
save_middle_model = False
if show_metric:
dataset_folder = ".. /input/generative-dog-images"
image_folder = ".. /input/generative-dog-images/all-dogs/all-dogs"
annotation_folder = ".. /input/generative-dog-images/annotation"
else:
dataset_folder = ".. /input"
image_folder = ".. /input/all-dogs"
annotation_folder = ".. /input/annotation"
else:
show_metric = True
show_animation = True
save_middle_model = False
dataset_folder = "."
image_folder = "all-dogs"
annotation_folder = "."
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )<init_hyperparams>
test = test.join(test_genre)
 test = test.join(test_cast)
 test = test.join(test_keywords)
 test = test.join(test_languages)
 test = test.join(test_prodcomp)
 test = test.join(test_prodcountries )
batch_size = 32
max_training_time = 32_300
if kernel_mode:
max_iterations = 3_000_000
else:
max_iterations = 112_000
code_size = 128
n_mlp = 8
n_critic = 1
class Args:
base_lr = 0.0015
lr = {8: 0.002, 16: 0.004, 32: 0.006, 64: 0.008}
batch = {8: 128, 16: 64, 32: 32, 64: 32}
phase = {8: 800_000, 16: 400_000, 32: 400_000, 64: 400_000}
init_size = 8
max_size = 64
mixing = True
loss = 'r1'
args = Args()
beta1 = 0.0
beta2 = 0.99
transform = transforms.Compose([
transforms.RandomHorizontalFlip(p=0.3),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5))
])
 if kernel_mode:
show_animation_iters = 2000
state_print_iters = 1000
else:
show_animation_iters = 2000
state_print_iters = 100
mifid_check_iters = 2000
mifid_check_images = 500
early_stopping_patience = 3
truncnorm_threshold = 1
if show_metric:
predict_batch_size = 25
else:
predict_batch_size = 50<feature_engineering>
add_datepart(train, 'release_date')
 add_datepart(test, 'release_date' )
start = time.time()<set_options>
cats_to_drop=['id','title','genres', 'cast', 'crew','Keywords','spoken_languages','production_companies','production_countries','homepage', 'belongs_to_collection','poster_path','imdb_id','original_language', 'original_title', 'overview','tagline','status']
train = train.drop(cats_to_drop, axis=1)
 train = train.drop('genres_tv_movie', axis=1)
 test = test.drop(cats_to_drop,axis=1 )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything()<load_pretrained>
train["revenue"]=np.log(train["revenue"] ).astype('float')
 train["budget"]=np.log(train["budget"]+0.1 ).astype('float')
 test["budget"]=np.log(test["budget"]+0.1 ).astype('float' )
def load_dataset_images(root, n_samples=25000, image_size=64):
IMG_EXTENSIONS =('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif',
'.tiff', '.webp')
 def is_valid_file(x):
return datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 required_transforms = transforms.Compose([
transforms.Resize(image_size),
transforms.CenterCrop(image_size),
])
 imgs = []
paths = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames)[:min(n_samples, 999999999999999)]:
path = os.path.join(root, fname)
 paths.append(path)
 for path in paths:
if is_valid_file(path):
img = datasets.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(
 dirname
for dirname in os.listdir(f'{annotation_folder}/Annotation/')
 if dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join(
 f'{annotation_folder}/Annotation/', annotation_dirname,
annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
bbox =(xmin, ymin, xmin + w, ymin + w)
 object_img = required_transforms(img.crop(bbox))
imgs.append(object_img)
 print(f"Number of cropped dog images: {len(imgs)}")
 return imgs
in_memory_images = load_dataset_images(image_folder )<train_model>
train.loc[train["runtime"].isnull() ,"runtime"]=train["runtime"].mode() [0]
train.loc[train["rating"].isnull() ,"rating"]=train["rating"].mode() [0]
train.loc[train["totalVotes"].isnull() ,"totalVotes"]=train["totalVotes"].mode() [0]
test.loc[test["runtime"].isnull() ,"runtime"]=train["runtime"].mode() [0]
test.loc[test["rating"].isnull() ,"rating"]=train["runtime"].mode() [0]
test.loc[test["totalVotes"].isnull() ,"totalVotes"]=train["totalVotes"].mode() [0]
test.loc[test["release_Year"].isnull() ,"release_Year"]=train["release_Year"].mode() [0]
test.loc[test["release_Month"].isnull() ,"release_Month"]=train["release_Month"].mode() [0]
test.loc[test["release_Week"].isnull() ,"release_Week"]=train["release_Week"].mode() [0]
test.loc[test["release_Day"].isnull() ,"release_Day"]=train["release_Day"].mode() [0]
test.loc[test["release_Dayofweek"].isnull() ,"release_Dayofweek"]=train["release_Dayofweek"].mode() [0]
test.loc[test["release_Dayofyear"].isnull() ,"release_Dayofyear"]=train["release_Dayofyear"].mode() [0]
class DataGenerator(torch.utils.data.Dataset):
def __init__(self,
dataset_images=None,
transform=None,
target_image_size=64):
self.dataset_images = dataset_images
self.transform = transform
self.target_image_size = target_image_size
self.samples = []
if self.target_image_size < 64:
print(f"Resizing images for DataGenerator to {target_image_size}x{target_image_size}...... ")
 resize_start_time = time.time()
required_transforms = transforms.Compose(
 [transforms.Resize(self.target_image_size)])
 for img in dataset_images:
self.samples.append(required_transforms(img))
print(f"Time spent on resizing images: " + \
f"{(time.time() - resize_start_time):.2f} seconds")
 else:
self.samples = self.dataset_images
def __getitem__(self, index):
sample = self.samples[index]
if self.transform is not None:
sample = self.transform(sample)
 return np.asarray(sample)
 def __len__(self):
return len(self.samples )<load_pretrained>
X = train.drop(['revenue'], axis=1)
 y = train['revenue']
X_test = test
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 repeated_folds = RepeatedStratifiedKFold(n_splits=10, n_repeats=20, random_state=42)
def sample_data(batch_size, image_size=8):
train_data = DataGenerator(
 in_memory_images, transform=transform, target_image_size=image_size)
 train_loader = torch.utils.data.DataLoader(
 train_data, shuffle=True, batch_size=batch_size, num_workers=4)
 return train_loader<train_model>
def train_model(X, X_test, y, params, folds, model_type='lgb', plot_feature_importance=False, averaging='usual', model=None):
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.loc[train_index], X.loc[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
if model_type == 'lgb':
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 model = lgb.train(params,
train_data,
num_boost_round=20000,
valid_sets = [train_data, valid_data],
verbose_eval=1000,
early_stopping_rounds = 200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X_train.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X_train.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X_train.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_train.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict_proba(X_valid ).reshape(-1,)
 score = mean_squared_error(y_valid, y_pred_valid)
 y_pred = model.predict_proba(X_test)[:, 1]
if model_type == 'glm':
model = sm.GLM(y_train, X_train, family=sm.families.Binomial())
 model_results = model.fit()
model_results.predict(X_test)
 y_pred_valid = model_results.predict(X_valid ).reshape(-1,)
 score = mean_squared_error(y_valid, y_pred_valid)
 y_pred = model_results.predict(X_test)
 if model_type == 'cat':
model = CatBoostClassifier(iterations=20000, learning_rate=0.1, loss_function='Logloss', eval_metric='AUC', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict_proba(X_valid)
 y_pred = model.predict_proba(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_squared_error(y_valid, y_pred_valid))
if averaging == 'usual':
prediction += y_pred
elif averaging == 'rank':
prediction += pd.Series(y_pred ).rank().values
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importance()
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance
return oof, prediction, scores
else:
return oof, prediction, scores
<init_hyperparams>
params = {'num_leaves': 16,
'min_data_in_leaf': 2,
'objective': 'regression',
'max_depth': 20,
'learning_rate': 0.008,
'boosting': 'gbdt',
'bagging_freq': 5,
'feature_fraction': 0.82,
'bagging_seed': 11,
'reg_alpha': 1.7,
'reg_lambda': 6,
'random_state': 42,
'metric': 'mse',
'verbosity': -1,
'subsample': 0.81,
'min_gain_to_split': 0.01,
'min_child_weight': 10,
'num_threads': 8}
oof_lgb, prediction_lgb, scores = train_model(X, X_test, y, params=params, folds=folds, model_type='lgb', plot_feature_importance=True )
def requires_grad(model, flag=True):
for p in model.parameters() :
p.requires_grad = flag
def accumulate(model1, model2, decay=0.999):
par1 = dict(model1.named_parameters())
 par2 = dict(model2.named_parameters())
 for k in par1.keys() :
par1[k].data.mul_(decay ).add_(1 - decay, par2[k].data)
 def adjust_lr(optimizer, lr):
for group in optimizer.param_groups:
mult = group.get('mult', 1)
 group['lr'] = lr * mult
def init_linear(linear):
init.xavier_normal(linear.weight)
 linear.bias.data.zero_()
def init_conv(conv, glu=True):
init.kaiming_normal(conv.weight)
 if conv.bias is not None:
conv.bias.data.zero_()<normalization>
xgb_params = {'eta': 0.1, 'max_depth': 3, 'subsample': 0.9, 'colsample_bytree': 0.9,
'objective': 'reg:linear', 'eval_metric': 'rmse', 'silent': True, 'nthread': 8}
oof_xgb, prediction_xgb, scores = train_model(X, X_test, y, params=xgb_params, folds=folds, model_type='xgb' )
<define_search_space><EOS> 
submission = pd.read_csv(f'{PATH}sample_submission.csv')
 submission['revenue'] = np.exp(0.50*(prediction_lgb+prediction_xgb))
submission.to_csv(f'submission.csv', index=False )