Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<define_search_model>
pd.set_option('max_columns', None)
 %matplotlib inline
plt.style.use('ggplot')
 stop = set(stopwords.words('english'))
py.init_notebook_mode(connected=True)
class Discriminator(nn.Module):
def __init__(self, fused=True):
super().__init__()
self.progression = nn.ModuleList([
ConvBlock(16, 32, 3, 1, downsample=True, fused=fused),
ConvBlock(32, 64, 3, 1, downsample=True, fused=fused),
ConvBlock(64, 128, 3, 1, downsample=True),
ConvBlock(128, 128, 3, 1, downsample=True),
ConvBlock(129, 128, 3, 1, 4, 0),
])
 self.from_rgb = nn.ModuleList([
EqualConv2d(3, 16, 1),
EqualConv2d(3, 32, 1),
EqualConv2d(3, 64, 1),
EqualConv2d(3, 128, 1),
EqualConv2d(3, 128, 1),
])
 self.n_layer = len(self.progression)
 self.linear = EqualLinear(code_size, 1)
 def forward(self, input, step=0, alpha=-1):
for i in range(step, -1, -1):
index = self.n_layer - i - 1
if i == step:
out = self.from_rgb[index](input)
 if i == 0:
out_std = torch.sqrt(out.var(0, unbiased=False)+ 1e-8)
 mean_std = out_std.mean()
mean_std = mean_std.expand(out.size(0), 1, 4, 4)
 out = torch.cat([out, mean_std], 1)
 out = self.progression[index](out)
 if i > 0:
if i == step and 0 <= alpha < 1:
skip_rgb = F.avg_pool2d(input, 2)
 skip_rgb = self.from_rgb[index + 1](skip_rgb)
 out =(1 - alpha)* skip_rgb + alpha * out
out = out.squeeze(2 ).squeeze(2)
 out = self.linear(out)
 return out<import_modules>
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
train = text_to_dict(train)
 test = text_to_dict(test )
import numpy as np
import os
import gzip, pickle
import tensorflow as tf
from scipy import linalg
import pathlib
import urllib
import warnings
from tqdm import tqdm
from PIL import Image<choose_model_class>
for i, e in enumerate(train['belongs_to_collection'][:5]):
print(i, e )
class KernelEvalException(Exception):
pass
def create_model_graph(pth):

 with tf.gfile.GFile(pth, 'rb')as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
 _ = tf.import_graph_def(graph_def, name='Pretrained_Net')
 def _get_model_layer(sess, model_name):
layername = model_params[model_name]['output_layer']
layer = sess.graph.get_tensor_by_name(layername)
 ops = layer.graph.get_operations()
for op_idx, op in enumerate(ops):
for o in op.outputs:
shape = o.get_shape()
if shape._dims != []:
shape = [s.value for s in shape]
new_shape = []
for j, s in enumerate(shape):
if s == 1 and j == 0:
new_shape.append(None)
 else:
new_shape.append(s)
 o.__dict__['_shape_val'] = tf.TensorShape(new_shape)
 return layer
def get_activations(images, sess, model_name, batch_size=32, verbose=False):

 inception_layer = _get_model_layer(sess, model_name)
 n_images = images.shape[0]
if batch_size > n_images:
print(
 "warning: batch size is bigger than the data size.setting batch size to data size" 
)
 batch_size = n_images
n_batches = n_images // batch_size + 1
pred_arr = np.empty(( n_images, model_params[model_name]['output_shape']))
for i in range(n_batches):
if verbose:
print("\rPropagating batch %d/%d" %(i + 1, n_batches),
end="",
flush=True)
 start = i * batch_size
if start + batch_size < n_images:
end = start + batch_size
else:
end = n_images
batch = images[start:end]
pred = sess.run(inception_layer,
{model_params[model_name]['input_layer']: batch})
 pred_arr[start:end] = pred.reshape(
 -1, model_params[model_name]['output_shape'])
 return pred_arr
def normalize_rows(x: np.ndarray):

 return np.nan_to_num(x / np.linalg.norm(x, ord=2, axis=1, keepdims=True))
def cosine_distance(features1, features2):
features1_nozero = features1[np.sum(features1, axis=1)!= 0]
features2_nozero = features2[np.sum(features2, axis=1)!= 0]
norm_f1 = normalize_rows(features1_nozero)
 norm_f2 = normalize_rows(features2_nozero)
 d = 1.0 - np.abs(np.matmul(norm_f1, norm_f2.T))
mean_min_d = np.mean(np.min(d, axis=1))
return mean_min_d
def distance_thresholding(d, eps):
if d < eps:
return d
else:
return 1
def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):

 mu1 = np.atleast_1d(mu1)
 mu2 = np.atleast_1d(mu2)
 sigma1 = np.atleast_2d(sigma1)
 sigma2 = np.atleast_2d(sigma2)
 assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"
diff = mu1 - mu2
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
 if not np.isfinite(covmean ).all() :
msg = "fid calculation produces singular product; adding %s to diagonal of cov estimates" % eps
warnings.warn(msg)
 offset = np.eye(sigma1.shape[0])* eps
covmean = linalg.sqrtm(( sigma1 + offset ).dot(sigma2 + offset))
if np.iscomplexobj(covmean):
if not np.allclose(np.diagonal(covmean ).imag, 0, atol=1e-3):
m = np.max(np.abs(covmean.imag))
raise ValueError("Imaginary component {}".format(m))
covmean = covmean.real
tr_covmean = np.trace(covmean)
 return diff.dot(diff)+ np.trace(sigma1)+ np.trace(
 sigma2)- 2 * tr_covmean
def calculate_activation_statistics(images,
sess,
model_name,
batch_size=32,
verbose=False):

 act = get_activations(images, sess, model_name, batch_size, verbose)
 mu = np.mean(act, axis=0)
 sigma = np.cov(act, rowvar=False)
 return mu, sigma, act
def _handle_path_memorization(path, sess, model_name, n_images,
is_checksize, is_check_png):
path = pathlib.Path(path)
 files = list(path.glob('*.jpg')) + list(path.glob('*.png'))
seed_everything()
np.random.shuffle(files)
 files = files[:n_images]
imsize = model_params[model_name]['imsize']
x = np.array([
np.array(
 img_read_checks(fn, imsize, is_checksize, imsize, is_check_png))
for fn in files
])
 m, s, features = calculate_activation_statistics(x, sess, model_name)
 del x
gc.collect()
return m, s, features
def img_read_checks(filename,
resize_to,
is_checksize=False,
check_imsize=64,
is_check_png=False):
im = Image.open(str(filename))
if is_checksize and im.size !=(check_imsize, check_imsize):
raise KernelEvalException('The images are not of size ' +
str(check_imsize))
if is_check_png and im.format != 'PNG':
raise KernelEvalException('Only PNG images should be submitted.')
 if resize_to is None:
return im
else:
return im.resize(( resize_to, resize_to), Image.ANTIALIAS)
 def calculate_kid_given_paths(paths, model_name, model_path,
n_images=1000,
feature_path=None):

 tf.reset_default_graph()
create_model_graph(str(model_path))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
 m1, s1, features1 = _handle_path_memorization(paths[0],
sess,
model_name,
n_images=n_images,
is_checksize=True,
is_check_png=True)
 if feature_path is None:
m2, s2, features2 = _handle_path_memorization(paths[1],
sess,
model_name,
n_images=n_images,
is_checksize=False,
is_check_png=False)
 else:
with np.load(feature_path)as f:
m2, s2, features2 = f['m'], f['s'], f['features']
fid_value = calculate_frechet_distance(m1, s1, m2, s2)
 distance = cosine_distance(features1, features2)
 return fid_value, distance<define_variables>
train['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
model_params = {
'Inception': {
'name': 'Inception',
'imsize': 64,
'output_layer': 'Pretrained_Net/pool_3:0',
'input_layer': 'Pretrained_Net/ExpandDims:0',
'output_shape': 2048,
'cosine_distance_eps': 0.1
}
}
user_images_unzipped_path = '.. /output_images'
images_path = [user_images_unzipped_path, image_folder]
if kernel_mode:
public_path = '.. /input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'
else:
public_path = 'classify_image_graph_def.pb'
fid_epsilon = 10e-15<statistical_test>
train['collection_name'] = train['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 train['has_collection'] = train['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 test['collection_name'] = test['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 test['has_collection'] = test['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 train = train.drop(['belongs_to_collection'], axis=1)
 test = test.drop(['belongs_to_collection'], axis=1 )
def compute_MiFID(n_images=1000):
seed_everything()
tf.set_random_seed(42)
 fid_value_public, distance_public = calculate_kid_given_paths(
 images_path, 'Inception', public_path, n_images)
 distance_public = distance_thresholding(
 distance_public, model_params['Inception']['cosine_distance_eps'])
 miFID = fid_value_public /(distance_public + fid_epsilon)
 print(
 f"FID_public: {fid_value_public}, distance_public: {distance_public} "
+ f"multiplied_public: {miFID}")
 return miFID<set_options>
for i, e in enumerate(train['genres'][:5]):
print(i, e )
generator = StyledGenerator(code_size, n_mlp ).cuda()
discriminator = Discriminator().cuda()
g_running = StyledGenerator(code_size, n_mlp ).cuda()
g_running.train(False )<choose_model_class>
print('Number of genres in films')
 train['genres'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
g_optimizer = optim.Adam(generator.generator.parameters() ,
lr=args.base_lr,
betas=(beta1, beta2))
g_optimizer.add_param_group({
'params': generator.style.parameters() ,
'lr': args.base_lr * 0.01,
'mult': 0.01,
})
 d_optimizer = optim.Adam(discriminator.parameters() ,
lr=args.base_lr,
betas=(beta1, beta2))
accumulate(g_running, generator, 0 )<init_hyperparams>
list_of_genres = list(train['genres'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values )
def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
 def save_models(model_name):
torch.save(
 {
'generator': generator.state_dict() ,
'discriminator': discriminator.state_dict() ,
'g_optimizer': g_optimizer.state_dict() ,
'd_optimizer': d_optimizer.state_dict() ,
'g_running': g_running.state_dict()
},
f'{model_name}.model', 
)
 print(
 f"Generator Model Size: {count_parameters(generator):,} trainable parameters" 
)
 print(
 f"Discriminator Model Size: {count_parameters(discriminator):,} trainable parameters"
)<set_options>
train['num_genres'] = train['genres'].apply(lambda x: len(x)if x != {} else 0)
 train['all_genres'] = train['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_genres = [m[0] for m in Counter([i for j in list_of_genres for i in j] ).most_common(15)]
for g in top_genres:
train['genre_' + g] = train['all_genres'].apply(lambda x: 1 if g in x else 0)
 test['num_genres'] = test['genres'].apply(lambda x: len(x)if x != {} else 0)
 test['all_genres'] = test['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_genres:
test['genre_' + g] = test['all_genres'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['genres'], axis=1)
 test = test.drop(['genres'], axis=1 )
torch.cuda.empty_cache()<init_hyperparams>
for i, e in enumerate(train['production_companies'][:5]):
print(i, e )
early_stopper = EarlyStoppingCriterion(patience=early_stopping_patience,
mode='min',
min_delta=1e-4)
 G_losses = []
D_losses = []
disc_loss_val = 0
gen_loss_val = 0
grad_loss_val = 0
gc.collect()
print("Starting Training Loop...")
 train_start_time = time.time()
alpha = 0
used_sample = 0
step = int(math.log2(args.init_size)) - 2
resolution = 4 * 2**step
phase = args.phase.get(resolution, 400_000)
 loader = sample_data(args.batch.get(resolution, batch_size),
image_size=resolution)
 data_loader = iter(loader)
 max_step = int(math.log2(args.max_size)) - 2
final_progress = False
adjust_lr(g_optimizer, args.lr.get(resolution, args.base_lr))
adjust_lr(d_optimizer, args.lr.get(resolution, args.base_lr))
requires_grad(generator, False)
 requires_grad(discriminator, True)
 print(f"Training with resolution {resolution}x{resolution}(" + \
f"step: {step}, batch_size: {args.batch.get(resolution, batch_size)}, " + \
f"Generator LR: {g_optimizer.state_dict() ['param_groups'][0]['lr']}, " + \
f"Style LR: {g_optimizer.state_dict() ['param_groups'][1]['lr']}, " + \
f"Discriminator LR: {d_optimizer.state_dict() ['param_groups'][0]['lr']})")
 go_ahead = True
for iters in range(max_iterations):
if not go_ahead:
break
gc.collect()
iter_start_time = time.time()
end = time.time()
if(end - start)> max_training_time:
print(
 f"Trained over {max_training_time:,} seconds, stopped at iteration {iters+1}" 
)
 go_ahead = False
break
discriminator.zero_grad()
alpha = min(1, 1 / phase *(used_sample + 1))
if resolution == args.init_size or final_progress:
alpha = 1
if used_sample > phase * 2:
used_sample = 0
step += 1
if step > max_step:
step = max_step
final_progress = True
else:
alpha = 0
resolution = 4 * 2**step
phase = args.phase.get(resolution, 400_000)
 del loader
gc.collect()
loader = sample_data(args.batch.get(resolution, batch_size),
image_size=resolution)
 data_loader = iter(loader)
 adjust_lr(g_optimizer, args.lr.get(resolution, args.base_lr))
adjust_lr(d_optimizer, args.lr.get(resolution, args.base_lr))
print(f"Training with resolution {resolution}x{resolution}(" + \
f"step: {step}, batch_size: {args.batch.get(resolution, batch_size)}, " + \
f"Generator LR: {g_optimizer.state_dict() ['param_groups'][0]['lr']}, " + \
f"Style LR: {g_optimizer.state_dict() ['param_groups'][1]['lr']}, " + \
f"Discriminator LR: {d_optimizer.state_dict() ['param_groups'][0]['lr']})")
 try:
real_image = next(data_loader)
 except(OSError, StopIteration):
data_loader = iter(loader)
 real_image = next(data_loader)
 used_sample += real_image.shape[0]
b_size = real_image.size(0)
 real_image = real_image.cuda()
if args.loss == 'wgan-gp':
real_predict = discriminator(real_image, step=step, alpha=alpha)
 real_predict = real_predict.mean() - 0.001 *(real_predict**2 ).mean() 
(-real_predict ).backward()
elif args.loss == 'r1':
real_image.requires_grad = True
real_predict = discriminator(real_image, step=step, alpha=alpha)
 real_predict = F.softplus(-real_predict ).mean()
real_predict.backward(retain_graph=True)
 grad_real = grad(outputs=real_predict.sum() ,
inputs=real_image,
create_graph=True)[0]
grad_penalty =(grad_real.view(grad_real.size(0),
-1 ).norm(2, dim=1)**2 ).mean()
grad_penalty = 10 / 2 * grad_penalty
grad_penalty.backward()
grad_loss_val = grad_penalty.item()
if args.mixing and random.random() < 0.9:
gen_in11, gen_in12, gen_in21, gen_in22 = torch.randn(
 4, b_size, code_size, device='cuda' ).chunk(4, 0)
 gen_in1 = [gen_in11.squeeze(0), gen_in12.squeeze(0)]
gen_in2 = [gen_in21.squeeze(0), gen_in22.squeeze(0)]
else:
gen_in1, gen_in2 = torch.randn(2, b_size, code_size,
device='cuda' ).chunk(2, 0)
 gen_in1 = gen_in1.squeeze(0)
 gen_in2 = gen_in2.squeeze(0)
 fake_image = generator(gen_in1, step=step, alpha=alpha)
 fake_predict = discriminator(fake_image, step=step, alpha=alpha)
 if args.loss == 'wgan-gp':
fake_predict = fake_predict.mean()
fake_predict.backward()
eps = torch.rand(b_size, 1, 1, 1 ).cuda()
x_hat = eps * real_image.data +(1 - eps)* fake_image.data
x_hat.requires_grad = True
hat_predict = discriminator(x_hat, step=step, alpha=alpha)
 grad_x_hat = grad(outputs=hat_predict.sum() ,
inputs=x_hat,
create_graph=True)[0]
grad_penalty =(
(grad_x_hat.view(grad_x_hat.size(0), -1 ).norm(2, dim=1)-
1)**2 ).mean()
grad_penalty = 10 * grad_penalty
grad_penalty.backward()
grad_loss_val = grad_penalty.item()
disc_loss_val =(real_predict - fake_predict ).item()
elif args.loss == 'r1':
fake_predict = F.softplus(fake_predict ).mean()
fake_predict.backward()
disc_loss_val =(real_predict + fake_predict ).item()
d_optimizer.step()
if(iters + 1)% n_critic == 0:
generator.zero_grad()
requires_grad(generator, True)
 requires_grad(discriminator, False)
 fake_image = generator(gen_in2, step=step, alpha=alpha)
 predict = discriminator(fake_image, step=step, alpha=alpha)
 if args.loss == 'wgan-gp':
loss = -predict.mean()
elif args.loss == 'r1':
loss = F.softplus(-predict ).mean()
gen_loss_val = loss.item()
loss.backward()
g_optimizer.step()
accumulate(g_running, generator)
 requires_grad(generator, False)
 requires_grad(discriminator, True)
 if show_animation and(iters + 1)% show_animation_iters == 0:
show_generated_img(iters)
 if(iters + 1)% state_print_iters == 0:
print(f"[{iters+1}/{max_iterations}] Phase Size:{phase}, " + \
f"Resolution: {4 * 2 ** step}, " + \
f"Loss_G: {gen_loss_val:.4f}, " + \
f"Loss_D: {disc_loss_val:.4f}, " + \
f"Grad: {grad_loss_val:.4f}, Alpha: {alpha:.5f}, Used Samples: {used_sample:,}")
 print(f"Time spent on iteration {iters+1}: " + \
f"{(time.time() - iter_start_time):.2f} seconds")
 G_losses.append(gen_loss_val)
 D_losses.append(disc_loss_val)
 if show_metric and(
(iters + 1)%
mifid_check_iters)== 0 and resolution == args.max_size:
generate_validation_images(threshold=truncnorm_threshold,
n_images=mifid_check_images)
 val_mifid = compute_MiFID(mifid_check_images)
 if early_stopper.step(val_mifid):
if early_stopper.is_improved:
best_score = val_mifid
best_iters = iters
print(
 f'MiFID Improved: {val_mifid:.4f}, saving model...... ' 
)
 else:
print('No improvements in this iteration')
 gc.collect()
else:
print(
 f'No more improvement....(Best MiFID: {best_score:.6f}, {best_iters+1} iterations)' 
)
 gc.collect()
break
if show_metric:
print(f'Best MiFID: {best_score:.6f}, {best_iters+1} iterations')
 else:
best_iters = iters
print(f"Total time spent on training: " + \
f"{(time.time() - train_start_time)/60:.2f} minutes " + \
f"({best_iters + 1} iterations, step: {step}, alpha: {alpha})" )<train_model>
print('Number of production companies in films')
 train['production_companies'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
<train_model>
train[train['production_companies'].apply(lambda x: len(x)if x != {} else 0)> 11]
if not show_metric:
final_model = f'stylegan_{best_iters+1}iters'
save_models(final_model)
 print(f"{final_model}.model saved." )<normalization>
img = Image.open(urlopen("https://image.tmdb.org/t/p/w600_and_h900_bestv2/5VKVaTJJsyDeOzY6fLcyTo1RA9g.jpg"))
img
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values<define_variables>
list_of_companies = list(train['production_companies'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values )
%%time
if not os.path.exists('.. /output_images'):
os.mkdir('.. /output_images')
 else:
shutil.rmtree('.. /output_images', ignore_errors=True)
 os.mkdir('.. /output_images')
 im_batch_size = 50
n_images = 10000
first_batch = []
for i_batch in range(0, n_images, im_batch_size):
z = truncated_normal(( im_batch_size, code_size),
threshold=truncnorm_threshold)
 gen_z = torch.from_numpy(z ).float().to(device)
 with torch.no_grad() :
gen_images = g_running(gen_z, step=step, alpha=alpha ).data.cpu()
for i_image in range(gen_images.size(0)) :
new_image = gen_images[i_image, :, :, :]
if i_batch == 0:
first_batch.append(new_image)
 save_image(new_image,
os.path.join('.. /output_images',
f'image_{i_batch+i_image:05d}.png'),
normalize=True,
range=(-1, 1))
generated_image_count = len([name for name in os.listdir('.. /output_images')])
 print(f"Number of generated images: {generated_image_count}")
 shutil.make_archive('images', 'zip', '.. /output_images' )<set_options>
train['num_companies'] = train['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 train['all_production_companies'] = train['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_companies = [m[0] for m in Counter([i for j in list_of_companies for i in j] ).most_common(30)]
for g in top_companies:
train['production_company_' + g] = train['all_production_companies'].apply(lambda x: 1 if g in x else 0)
 test['num_companies'] = test['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 test['all_production_companies'] = test['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_companies:
test['production_company_' + g] = test['all_production_companies'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['production_companies', 'all_production_companies'], axis=1)
 test = test.drop(['production_companies', 'all_production_companies'], axis=1 )
%%time
show_generated_img_all(64 )<import_modules>
for i, e in enumerate(train['production_countries'][:5]):
print(i, e )
import shutil
import os
import random
from time import time
import matplotlib.pyplot as plt
import numpy as np
import torch
from torch.nn.utils import spectral_norm
from torch.utils.data import Dataset
import torchvision
import xml.etree.ElementTree as ET
from tqdm import tqdm_notebook as tqdm
from scipy.stats import truncnorm 
<define_variables>
print('Number of production countries in films')
 train['production_countries'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
CHANNEL = {'RGB': 3, 'BW': 1}
IMG_CHANNEL = CHANNEL['RGB']
DATA_LOCATION = '.. /input/all-dogs/all-dogs/'
BATCH_SIZE = 32
LATENT_DIM = 128
CONV_DEPTHS_G = 32
CONV_DEPTHS_D = 48
LEARNING_RATE_G = 0.0003
LEARNING_RATE_D = 0.0001
BETA_1 = 0.5
T0_interval = 200
EPOCHS = 600
NUM_CLASSES = 10
TIME_FOR_TRAIN = 25000 
<set_options>
list_of_countries = list(train['production_countries'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_countries for i in j] ).most_common(25 )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
<load_pretrained>
train['num_countries'] = train['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 train['all_countries'] = train['production_countries'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_countries = [m[0] for m in Counter([i for j in list_of_countries for i in j] ).most_common(25)]
for g in top_countries:
train['production_country_' + g] = train['all_countries'].apply(lambda x: 1 if g in x else 0)
 test['num_countries'] = test['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 test['all_countries'] = test['production_countries'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_countries:
test['production_country_' + g] = test['all_countries'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['production_countries', 'all_countries'], axis=1)
 test = test.drop(['production_countries', 'all_countries'], axis=1 )
class DataGenerator(Dataset):
def __init__(self, directory, transform=None, n_samples=np.inf):
self.directory = directory
self.transform = transform
self.n_samples = n_samples
self.samples = self._load_subfolders_images(directory)
 if len(self.samples)== 0:
raise RuntimeError("Found 0 files in subfolders of: {}".format(directory))
def _load_subfolders_images(self, root):
IMG_EXTENSIONS =(
 '.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
 def is_valid_file(x):
return torchvision.datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 required_transforms = torchvision.transforms.Compose([
torchvision.transforms.Resize(64),
torchvision.transforms.CenterCrop(64),
])
 imgs = []
paths = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames)[:min(self.n_samples, 999999999999999)]:
path = os.path.join(root, fname)
 paths.append(path)
 for path in paths:
if is_valid_file(path):
img = torchvision.datasets.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(
 dirname for dirname in os.listdir('.. /input/annotation/Annotation/')if
dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join('.. /input/annotation/Annotation/',
annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
bbox =(xmin, ymin, xmin + w, ymin + w)
 object_img = required_transforms(img.crop(bbox))
imgs.append(object_img)
 return imgs
def __getitem__(self, index):
sample = self.samples[index]
if self.transform is not None:
sample = self.transform(sample)
 return np.asarray(sample)
 def __len__(self):
return len(self.samples)
 transform = torchvision.transforms.Compose([torchvision.transforms.RandomHorizontalFlip(p=0.1),
torchvision.transforms.ToTensor() ,
torchvision.transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 data_gen = DataGenerator(DATA_LOCATION, transform=transform, n_samples=25000)
 train_loader = torch.utils.data.DataLoader(data_gen, shuffle=True, batch_size=BATCH_SIZE, num_workers=4 )<normalization>
for i, e in enumerate(train['spoken_languages'][:5]):
print(i, e )
class PixelwiseNorm(torch.nn.Module):
def __init__(self):
super(PixelwiseNorm, self ).__init__()
def forward(self, x, alpha=1e-8):

 y = x.pow(2.).mean(dim=1, keepdim=True ).add(alpha ).sqrt()
y = x / y
return y
class Generator(torch.nn.Module):
def __init__(self, latent_dim, nfeats, nchannels):
super(Generator, self ).__init__()
self.conv1 = spectral_norm(torch.nn.ConvTranspose2d(latent_dim, nfeats * 8, 4, 1, 0, bias=False))
self.conv2 = spectral_norm(torch.nn.ConvTranspose2d(nfeats * 8, nfeats * 8, 4, 2, 1, bias=False))
self.conv3 = spectral_norm(torch.nn.ConvTranspose2d(nfeats * 8, nfeats * 4, 4, 2, 1, bias=False))
self.conv4 = spectral_norm(torch.nn.ConvTranspose2d(nfeats * 4, nfeats * 2, 4, 2, 1, bias=False))
self.conv5 = spectral_norm(torch.nn.ConvTranspose2d(nfeats * 2, nfeats, 4, 2, 1, bias=False))
self.conv6 = spectral_norm(torch.nn.ConvTranspose2d(nfeats, nchannels, 3, 1, 1, bias=False))
self.pixnorm = PixelwiseNorm()
def forward(self, x):
x = torch.nn.functional.leaky_relu(self.conv1(x))
x = self.pixnorm(x)
 x = torch.nn.functional.leaky_relu(self.conv2(x))
x = torch.nn.Dropout(0.05 )(x)
 x = self.pixnorm(x)
 x = torch.nn.functional.leaky_relu(self.conv3(x))
x = self.pixnorm(x)
 x = torch.nn.functional.leaky_relu(self.conv4(x))
x = torch.nn.Dropout(0.05 )(x)
 x = self.pixnorm(x)
 x = torch.nn.functional.leaky_relu(self.conv5(x))
x = self.pixnorm(x)
 x = torch.tanh(self.conv6(x))
return x 
<categorify>
print('Number of spoken languages in films')
 train['spoken_languages'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
class MinibatchStdDev(torch.nn.Module):

 def __init__(self):

 super(MinibatchStdDev, self ).__init__()
def forward(self, x, alpha=1e-8):

 batch_size, _, height, width = x.shape
y = x - x.mean(dim=0, keepdim=True)
 y = torch.sqrt(y.pow(2.).mean(dim=0, keepdim=False)+ alpha)
 y = y.mean().view(1, 1, 1, 1)
 y = y.repeat(batch_size, 1, height, width)
 y = torch.cat([x, y], 1)
 return y<define_search_model>
list_of_languages = list(train['spoken_languages'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_languages for i in j] ).most_common(15 )
class Discriminator(torch.nn.Module):
def __init__(self, nchannels, nfeats):
super(Discriminator, self ).__init__()
self.conv1 = torch.nn.Conv2d(nchannels, nfeats, 4, 2, 1, bias=False)
 self.conv2 = spectral_norm(torch.nn.Conv2d(nfeats, nfeats * 2, 4, 2, 1, bias=False))
self.bn2 = torch.nn.BatchNorm2d(nfeats * 2)
 self.conv3 = spectral_norm(torch.nn.Conv2d(nfeats * 2, nfeats * 4, 4, 2, 1, bias=False))
self.bn3 = torch.nn.BatchNorm2d(nfeats * 4)
 self.conv4 = spectral_norm(torch.nn.Conv2d(nfeats * 4, nfeats * 8, 4, 2, 1, bias=False))
self.bn4 = torch.nn.MaxPool2d(2)
 self.batch_discriminator = MinibatchStdDev()
self.conv5 = spectral_norm(torch.nn.Conv2d(nfeats * 8 + 1, 1, 2, 1, 0, bias=False))
def forward(self, x):
x = torch.nn.functional.leaky_relu(self.conv1(x), 0.2)
 x = torch.nn.functional.leaky_relu(self.bn2(self.conv2(x)) , 0.2)
 x = torch.nn.functional.leaky_relu(self.bn3(self.conv3(x)) , 0.2)
 x = torch.nn.functional.leaky_relu(self.bn4(self.conv4(x)) , 0.2)
 x = self.batch_discriminator(x)
 x = self.conv5(x)
 x = x.view(x.size(0), -1)
 return x 
<compute_test_metric>
train['num_languages'] = train['spoken_languages'].apply(lambda x: len(x)if x != {} else 0)
 train['all_languages'] = train['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_languages = [m[0] for m in Counter([i for j in list_of_languages for i in j] ).most_common(30)]
for g in top_languages:
train['language_' + g] = train['all_languages'].apply(lambda x: 1 if g in x else 0)
 test['num_languages'] = test['spoken_languages'].apply(lambda x: len(x)if x != {} else 0)
 test['all_languages'] = test['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_languages:
test['language_' + g] = test['all_languages'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['spoken_languages', 'all_languages'], axis=1)
 test = test.drop(['spoken_languages', 'all_languages'], axis=1 )
def mse_loss(input, target):
return torch.sum(( input - target)**2)/ input.data.nelement()
gan_loss = mse_loss
fixed_noise = torch.randn(25, LATENT_DIM, 1, 1, device=device)
<compute_test_metric>
for i, e in enumerate(train['Keywords'][:5]):
print(i, e )
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values<train_model>
print('Number of Keywords in films')
 train['Keywords'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(10 )
def train_module(epochs):
step = 0
start = time()
for epoch in range(epochs):
for ii,(real_images)in enumerate(train_loader):
end = time()
if(end - start)> TIME_FOR_TRAIN:
break
netD.zero_grad()
real_images = real_images.to(device)
 batch_size = real_images.size(0)
 labels = torch.full(( batch_size, 1), real_label, device=device)
 outputR = netD(real_images)
 D_x = outputR.mean().item()
noise = torch.randn(batch_size, LATENT_DIM,1, 1, device=device)
 fake = netG(noise)
 outputF = netD(fake.detach())
 D_G_z1 = outputF.mean().item()
errD = gan_loss(outputR, torch.mean(outputF)+ labels)+ gan_loss(outputF, torch.mean(outputR)- labels)
 errD.backward(retain_graph=True)
 optimizerD.step()
netG.zero_grad()
labels.fill_(real_label)
 outputF = netD(fake)
 errG = gan_loss(outputR, torch.mean(outputF)- labels)+ gan_loss(outputF, torch.mean(outputR)+ labels)
 errG.backward(retain_graph=True)
 D_G_z2 = outputF.mean().item()
optimizerG.step()
if step % 500 == 0:
print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)) : %.4f / %.4f'
%(epoch + 1, epochs, ii, len(train_loader),
errD.item() , errG.item() , D_x, D_G_z1, D_G_z2))
valid_image = netG(fixed_noise)
 step += 1
lr_schedulerG.step(epoch)
 lr_schedulerD.step(epoch)
 if epoch % 5 == 0:
show_generated_img()
print(end - start)
 if(end - start)> TIME_FOR_TRAIN:
break<choose_model_class>
list_of_keywords = list(train['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 plt.figure(figsize =(16, 12))
text = ' '.join(['_'.join(i.split(' ')) for j in list_of_keywords for i in j])
 wordcloud = WordCloud(max_font_size=None, background_color='black', collocations=False,
width=1200, height=1000 ).generate(text)
 plt.imshow(wordcloud)
 plt.title('Top keywords')
 plt.axis("off")
 plt.show()
netG = Generator(LATENT_DIM, CONV_DEPTHS_G, IMG_CHANNEL ).to(device)
 netD = Discriminator(3, CONV_DEPTHS_D ).to(device)
 LEARNING_RATE_G = 0.003
LEARNING_RATE_D = 0.001
optimizerD = torch.optim.Adam(netD.parameters() , lr=LEARNING_RATE_G, betas=(BETA_1, 0.999))
optimizerG = torch.optim.Adam(netG.parameters() , lr=LEARNING_RATE_D, betas=(BETA_1, 0.999))
lr_schedulerG = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerG,
T_0=EPOCHS // T0_interval, eta_min=0.00005)
 lr_schedulerD = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerD,
T_0=EPOCHS // T0_interval, eta_min=0.00005)
 fixed_noise = torch.randn(25, LATENT_DIM, 1, 1, device=device)
 real_label = 0.5
fake_label = 0.0
batch_size = train_loader.batch_size
train_module(EPOCHS )<import_modules>
train['num_Keywords'] = train['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 train['all_Keywords'] = train['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_keywords = [m[0] for m in Counter([i for j in list_of_keywords for i in j] ).most_common(30)]
for g in top_keywords:
train['keyword_' + g] = train['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 test['num_Keywords'] = test['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 test['all_Keywords'] = test['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_keywords:
test['keyword_' + g] = test['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['Keywords', 'all_Keywords'], axis=1)
 test = test.drop(['Keywords', 'all_Keywords'], axis=1 )
batch_size = 64<define_variables>
for i, e in enumerate(train['cast'][:1]):
print(i, e )
start = time()<set_options>
print('Number of casted persons in films')
 train['cast'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(10 )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<load_pretrained>
list_of_cast_names = list(train['cast'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_names for i in j] ).most_common(15 )
class DataGenerator(Dataset):
def __init__(self, directory, transform=None, n_samples=np.inf):
self.directory = directory
self.transform = transform
self.n_samples = n_samples
self.samples = self._load_subfolders_images(directory)
 if len(self.samples)== 0:
raise RuntimeError("Found 0 files in subfolders of: {}".format(directory))
def _load_subfolders_images(self, root):
IMG_EXTENSIONS =(
 '.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
 def is_valid_file(x):
return torchvision.datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 required_transforms = torchvision.transforms.Compose([
torchvision.transforms.Resize(64),
torchvision.transforms.CenterCrop(64),
])
 imgs = []
paths = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames)[:min(self.n_samples, 999999999999999)]:
path = os.path.join(root, fname)
 paths.append(path)
 for path in paths:
if is_valid_file(path):
img = dset.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(
 dirname for dirname in os.listdir('.. /input/annotation/Annotation/')if
dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join('.. /input/annotation/Annotation/',
annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
bbox =(xmin, ymin, xmin+w, ymin+w)
 object_img = required_transforms(img.crop(bbox))
imgs.append(object_img)
 return imgs
def __getitem__(self, index):
sample = self.samples[index]
if self.transform is not None:
sample = self.transform(sample)
 return np.asarray(sample)
 def __len__(self):
return len(self.samples )<categorify>
list_of_cast_genders = list(train['cast'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_genders for i in j] ).most_common()
%%time
database = '.. /input/all-dogs/all-dogs/'
transform = transforms.Compose([transforms.RandomHorizontalFlip(p=0.3),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 train_data = DataGenerator(database, transform=transform,n_samples=25000)
 train_loader = torch.utils.data.DataLoader(train_data, shuffle=True, batch_size=batch_size, num_workers = 4 )<normalization>
list_of_cast_characters = list(train['cast'].apply(lambda x: [i['character'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_characters for i in j] ).most_common(15 )
class PixelwiseNorm(nn.Module):
def __init__(self):
super(PixelwiseNorm, self ).__init__()
def forward(self, x, alpha=1e-8):

 y = x.pow(2.).mean(dim=1, keepdim=True ).add(alpha ).sqrt()
y = x / y
return y<categorify>
train['num_cast'] = train['cast'].apply(lambda x: len(x)if x != {} else 0)
 top_cast_names = [m[0] for m in Counter([i for j in list_of_cast_names for i in j] ).most_common(15)]
for g in top_cast_names:
train['cast_name_' + g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 train['genders_0_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['genders_1_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['genders_2_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
top_cast_characters = [m[0] for m in Counter([i for j in list_of_cast_characters for i in j] ).most_common(15)]
for g in top_cast_characters:
train['cast_character_' + g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 test['num_cast'] = test['cast'].apply(lambda x: len(x)if x != {} else 0)
 for g in top_cast_names:
test['cast_name_' + g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0)
 test['genders_0_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['genders_1_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['genders_2_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
for g in top_cast_characters:
test['cast_character_' + g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0)
 train = train.drop(['cast'], axis=1)
 test = test.drop(['cast'], axis=1 )
class MinibatchStdDev(th.nn.Module):

 def __init__(self):

 super(MinibatchStdDev, self ).__init__()
def forward(self, x, alpha=1e-8):

 batch_size, _, height, width = x.shape
y = x - x.mean(dim=0, keepdim=True)
 y = th.sqrt(y.pow(2.).mean(dim=0, keepdim=False)+ alpha)
 y = y.mean().view(1, 1, 1, 1)
 y = y.repeat(batch_size,1, height, width)
 y = th.cat([x, y], 1)
 return y
class GeneralizedDropOut(nn.Module):
def __init__(self, mode='mul', strength=0.4, axes=(0,1), normalize=False):
super(GeneralizedDropOut, self ).__init__()
self.mode = mode.lower()
assert self.mode in ['mul', 'drop', 'prop'], 'Invalid GDropLayer mode'%mode
self.strength = strength
self.axes = [axes] if isinstance(axes, int)else list(axes)
 self.normalize = normalize
self.gain = None
def forward(self, x, deterministic=False):
if deterministic or not self.strength:
return x
rnd_shape = [s if axis in self.axes else 1 for axis, s in enumerate(x.size())]
if self.mode == 'drop':
p = 1 - self.strength
rnd = np.random.binomial(1, p=p, size=rnd_shape)/ p
elif self.mode == 'mul':
rnd =(1 + self.strength)** np.random.normal(size=rnd_shape)
 else:
coef = self.strength * x.size(1)** 0.5
rnd = np.random.normal(size=rnd_shape)* coef + 1
if self.normalize:
rnd = rnd / np.linalg.norm(rnd, keepdims=True)
 rnd = Variable(torch.from_numpy(rnd ).type(x.data.type()))
if x.is_cuda:
rnd = rnd.cuda()
return x * rnd
def __repr__(self):
param_str = '(mode = %s, strength = %s, axes = %s, normalize = %s)' %(self.mode, self.strength, self.axes, self.normalize)
 return self.__class__.__name__ + param_str<normalization>
print('Number of casted persons in films')
 train['crew'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(10 )
def l2normalize(v, eps=1e-12):
return v /(v.norm() + eps)
 class SpectralNorm(nn.Module):
def __init__(self, module, name='weight', power_iterations=1):
super(SpectralNorm, self ).__init__()
self.module = module
self.name = name
self.power_iterations = power_iterations
def _update_u_v(self):
if not self._made_params() :
self._make_params()
w = getattr(self.module, self.name)
 u = getattr(self.module, self.name + "_u")
 height = w.data.shape[0]
for _ in range(self.power_iterations):
v = l2normalize(torch.mv(torch.t(w.view(height,-1 ).data), u))
u = l2normalize(torch.mv(w.view(height,-1 ).data, v))
setattr(self.module, self.name + "_u", u)
 w.data = w.data / torch.dot(u, torch.mv(w.view(height,-1 ).data, v))
def _made_params(self):
try:
u = getattr(self.module, self.name + "_u")
 return True
except AttributeError:
return False
def _make_params(self):
w = getattr(self.module, self.name)
 height = w.data.shape[0]
width = w.view(height, -1 ).data.shape[1]
u = l2normalize(w.data.new(height ).normal_(0, 1))
self.module.register_buffer(self.name + "_u", u)
 def forward(self, *args):
self._update_u_v()
return self.module.forward(*args )<define_search_model>
list_of_crew_names = list(train['crew'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_names for i in j] ).most_common(15 )
class Generator(nn.Module):
def __init__(self, nz, nfeats, nchannels):
super(Generator, self ).__init__()
self.conv1 = spectral_norm(nn.ConvTranspose2d(nz, nfeats * 8, 4, 1, 0, bias=False))
self.bn1 = nn.BatchNorm2d(nfeats * 8)
 self.conv2 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 8, 4, 2, 1, bias=False))
self.bn2 = nn.BatchNorm2d(nfeats * 8)
 self.conv3 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 4, 4, 2, 1, bias=False))
self.bn3 = nn.BatchNorm2d(nfeats * 4)
 self.conv4 = spectral_norm(nn.ConvTranspose2d(nfeats * 4, nfeats * 2, 4, 2, 1, bias=False))
self.bn4 = nn.BatchNorm2d(nfeats * 2)
 self.conv5 = spectral_norm(nn.ConvTranspose2d(nfeats * 2, nfeats, 4, 2, 1, bias=False))
self.bn5 = nn.BatchNorm2d(nfeats)
 self.conv6 = spectral_norm(nn.ConvTranspose2d(nfeats, nchannels, 3, 1, 1, bias=False))
self.pixnorm = PixelwiseNorm()
def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = F.leaky_relu(x, 0.05)
 x = self.pixnorm(x)
 x = self.conv2(x)
 x = self.bn2(x)
 x = F.leaky_relu(x, 0.05)
 x = self.pixnorm(x)
 x = self.conv3(x)
 x = self.bn3(x)
 x = F.leaky_relu(x, 0.05)
 x = self.pixnorm(x)
 x = self.conv4(x)
 x = self.bn4(x)
 x = F.leaky_relu(x, 0.05)
 x = self.pixnorm(x)
 x = self.conv5(x)
 x = self.bn5(x)
 x = F.leaky_relu(x, 0.05)
 x = self.pixnorm(x)
 x = self.conv6(x)
 x = torch.tanh(x)
 return x
class Discriminator(nn.Module):
def __init__(self, nchannels, nfeats):
super(Discriminator, self ).__init__()
self.conv1 = nn.Conv2d(nchannels, nfeats, 4, 2, 1, bias=False)
 self.conv2 = spectral_norm(nn.Conv2d(nfeats, nfeats * 2, 4, 2, 1, bias=False))
self.bn2 = nn.BatchNorm2d(nfeats * 2)
 self.conv3 = spectral_norm(nn.Conv2d(nfeats * 2, nfeats * 4, 4, 2, 1, bias=False))
self.gdo3 = GeneralizedDropOut(mode='prop', strength=0.0)
 self.bn3 = nn.BatchNorm2d(nfeats * 4)
 self.conv4 = spectral_norm(nn.Conv2d(nfeats * 4, nfeats * 8, 4, 2, 1, bias=False))
self.bn4 = nn.MaxPool2d(2)
 self.conv5 = spectral_norm(nn.Conv2d(nfeats * 8 +1, 1, 2, 1, 0, bias=False))
self.avg_pool2d = nn.AvgPool2d(kernel_size=2)
 self.pixnorm = PixelwiseNorm()
self.batch_discriminator = MinibatchStdDev()
def forward(self, x):
x = self.conv1(x)
 x = F.leaky_relu(x, 0.2)
 x = self.conv2(x)
 x = self.bn2(x)
 x = F.leaky_relu(x, 0.2)
 x = self.pixnorm(x)
 x = self.conv3(x)
 x = self.gdo3(x)
 x = self.bn3(x)
 x = F.leaky_relu(x, 0.2)
 x = self.pixnorm(x)
 x = self.conv4(x)
 x = self.bn4(x)
 x = F.leaky_relu(x, 0.2)
 x = self.pixnorm(x)
 x = self.batch_discriminator(x)
 x = self.conv5(x)
 x= self.avg_pool2d(x)
 x = torch.sigmoid(x)
 return x.view(-1, 1)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 lr = 0.0003
lr_d = 0.0001
beta1 = 0.5
epochs = 1400
nz = 256
netG = Generator(nz, 64, 3 ).to(device)
 netD = Discriminator(3, 64 ).to(device)
 criterion = nn.BCELoss()
optimizerD = optim.Adam(netD.parameters() , lr=lr, betas=(beta1, 0.999))
optimizerG = optim.Adam(netG.parameters() , lr=lr_d, betas=(beta1, 0.999))
lr_schedulerG = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerG,
T_0=epochs//200, eta_min=0.00005)
 lr_schedulerD = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerD,
T_0=epochs//200, eta_min=0.00005)
 fixed_noise = torch.randn(25, nz, 1, 1, device=device)
 real_label = 0.99
fake_label = 0.01
batch_size = train_loader.batch_size
step = 0
for epoch in range(epochs):
for ii,(real_images)in tqdm(enumerate(train_loader), total=len(train_loader)) :
end = time()
if(end - start)> 32000 :
break
netD.zero_grad()
real_images = real_images.to(device)
 batch_size = real_images.size(0)
 labels = torch.full(( batch_size, 1), real_label, device=device)+ np.random.uniform(-0.1, 0.1)
 output = netD(real_images)
 errD_real = criterion(output, labels)
 errD_real.backward()
D_x = output.mean().item()
noise = torch.randn(batch_size, nz, 1, 1, device=device)
 fake = netG(noise)
 labels.fill_(fake_label)+ np.random.uniform(0, 0.2)
 output = netD(fake.detach())
 errD_fake = criterion(output, labels)
 errD_fake.backward()
D_G_z1 = output.mean().item()
errD = errD_real + errD_fake
optimizerD.step()
netG.zero_grad()
labels.fill_(real_label)
 output = netD(fake)
 errG = criterion(output, labels)
 errG.backward()
D_G_z2 = output.mean().item()
optimizerG.step()
if step % 500 == 0:
print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)) : %.4f / %.4f'
%(epoch + 1, epochs, ii, len(train_loader),
errD.item() , errG.item() , D_x, D_G_z1, D_G_z2))
valid_image = netG(fixed_noise)
 step += 1
lr_schedulerG.step(errG.item())
 lr_schedulerD.step(errD.item())
 if epoch % 5 == 0:
show_generated_img()
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values
if not os.path.exists('.. /output_images'):
os.mkdir('.. /output_images')
 im_batch_size = 50
n_images=10000
for i_batch in range(0, n_images, im_batch_size):
z = truncated_normal(( im_batch_size, nz, 1, 1), threshold=1)
 gen_z = torch.from_numpy(z ).float().to(device)
 gen_images = netG(gen_z)
 images = gen_images.to("cpu" ).clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
 for i_image in range(gen_images.size(0)) :
save_image(( gen_images[i_image, :, :, :] +1.0)/2.0, os.path.join('.. /output_images', f'image_{i_batch+i_image:05d}.png'))
shutil.make_archive('images', 'zip', '.. /output_images' )<import_modules>
list_of_crew_jobs = list(train['crew'].apply(lambda x: [i['job'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_jobs for i in j] ).most_common(15 )
import datetime
import gc
import json
import math
import os
import time
import shutil
import sys
from functools import partial
from numba import jit, njit
from PIL import Image
from tqdm import tqdm, trange
import cv2
import numpy as np
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
from torch.nn import init
import torch.optim as optim
import torch.nn.functional as F
from torch.nn import Parameter as P
import torch.utils.data as data
from torch.utils.data import DataLoader
import torchvision
from torchvision import datasets
import torchvision.transforms as T
import albumentations as A
from albumentations.pytorch import ToTensor
import xml.etree.ElementTree as ET<define_variables>
list_of_crew_genders = list(train['crew'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_genders for i in j] ).most_common(15 )
INPUT_PATH = '.. /input/all-dogs'
ANNOT_PATH = '.. /input/annotation/Annotation'
OUT_DIR = '.. /output_images'
IMG_SIZE = 64
IMG_SIZE_2 = IMG_SIZE * 2
IMG_CHANNELS = 3
IMG_EXTENSIONS =('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp' )<compute_test_metric>
list_of_crew_departments = list(train['crew'].apply(lambda x: [i['department'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_departments for i in j] ).most_common(14 )
def proj(x, y):
return torch.mm(y, x.t())* y / torch.mm(y, y.t())
 def gram_schmidt(x, ys):
for y in ys:
x = x - proj(x, y)
 return x
def power_iteration(W, u_, update=True, eps=1e-12):
us, vs, svs = [], [], []
for i, u in enumerate(u_):
with torch.no_grad() :
v = torch.matmul(u, W)
 v = F.normalize(gram_schmidt(v, vs), eps=eps)
 vs += [v]
u = torch.matmul(v, W.t())
 u = F.normalize(gram_schmidt(u, us), eps=eps)
 us += [u]
if update:
u_[i][:] = u
svs += [torch.squeeze(torch.matmul(torch.matmul(v, W.t()), u.t())) ]
return svs, us, vs
class identity(nn.Module):
def forward(self, input):
return input
class SN(object):
def __init__(self, num_svs, num_itrs, num_outputs, transpose=False, eps=1e-12):
self.num_itrs = num_itrs
self.num_svs = num_svs
self.transpose = transpose
self.eps = eps
for i in range(self.num_svs):
self.register_buffer('u%d' % i, torch.randn(1, num_outputs))
self.register_buffer('sv%d' % i, torch.ones(1))
@property
def u(self):
return [getattr(self, 'u%d' % i)for i in range(self.num_svs)]
@property
def sv(self):
return [getattr(self, 'sv%d' % i)for i in range(self.num_svs)]
def W_(self):
W_mat = self.weight.view(self.weight.size(0), -1)
 if self.transpose:
W_mat = W_mat.t()
for _ in range(self.num_itrs):
svs, us, vs = power_iteration(W_mat, self.u, update=self.training, eps=self.eps)
 if self.training:
with torch.no_grad() :
for i, sv in enumerate(svs):
self.sv[i][:] = sv
return self.weight / svs[0]
class SNConv2d(nn.Conv2d, SN):
def __init__(self, in_channels, out_channels, kernel_size, stride=1,
padding=0, dilation=1, groups=1, bias=True,
num_svs=1, num_itrs=1, eps=1e-12):
nn.Conv2d.__init__(self, in_channels, out_channels, kernel_size, stride,
padding, dilation, groups, bias)
 SN.__init__(self, num_svs, num_itrs, out_channels, eps=eps)
 def forward(self, x):
return F.conv2d(x, self.W_() , self.bias, self.stride,
self.padding, self.dilation, self.groups)
 class SNLinear(nn.Linear, SN):
def __init__(self, in_features, out_features, bias=True,
num_svs=1, num_itrs=1, eps=1e-12):
nn.Linear.__init__(self, in_features, out_features, bias)
 SN.__init__(self, num_svs, num_itrs, out_features, eps=eps)
 def forward(self, x):
return F.linear(x, self.W_() , self.bias)
 class SNEmbedding(nn.Embedding, SN):
def __init__(self, num_embeddings, embedding_dim, padding_idx=None,
max_norm=None, norm_type=2, scale_grad_by_freq=False,
sparse=False, _weight=None,
num_svs=1, num_itrs=1, eps=1e-12):
nn.Embedding.__init__(self, num_embeddings, embedding_dim, padding_idx,
max_norm, norm_type, scale_grad_by_freq,
sparse, _weight)
 SN.__init__(self, num_svs, num_itrs, num_embeddings, eps=eps)
 def forward(self, x):
return F.embedding(x, self.W_())
 class Attention(nn.Module):
def __init__(self, ch, which_conv=SNConv2d, name='attention'):
super(Attention, self ).__init__()
self.ch = ch
self.which_conv = which_conv
self.theta = self.which_conv(self.ch, self.ch // 4, kernel_size=1, padding=0, bias=False)
 self.phi = self.which_conv(self.ch, self.ch // 4, kernel_size=1, padding=0, bias=False)
 self.g = self.which_conv(self.ch, self.ch // 2, kernel_size=1, padding=0, bias=False)
 self.o = self.which_conv(self.ch // 2, self.ch, kernel_size=1, padding=0, bias=False)
 self.gamma = P(torch.tensor(0.) , requires_grad=True)
 def forward(self, x, y=None):
theta = self.theta(x)
 phi = F.max_pool2d(self.phi(x), [2, 2])
 g = F.max_pool2d(self.g(x), [2, 2])
 theta = theta.view(-1, self.ch // 4, x.shape[2] * x.shape[3])
 phi = phi.view(-1, self.ch // 4, x.shape[2] * x.shape[3] // 4)
 g = g.view(-1, self.ch // 2, x.shape[2] * x.shape[3] // 4)
 beta = F.softmax(torch.bmm(theta.transpose(1, 2), phi), -1)
 o = self.o(torch.bmm(g, beta.transpose(1, 2)).view(-1, self.ch // 2, x.shape[2], x.shape[3]))
return self.gamma * o + x
def fused_bn(x, mean, var, gain=None, bias=None, eps=1e-5):
scale = torch.rsqrt(var + eps)
 if gain is not None:
scale = scale * gain
shift = mean * scale
if bias is not None:
shift = shift - bias
return x * scale - shift
def manual_bn(x, gain=None, bias=None, return_mean_var=False, eps=1e-5):
float_x = x.float()
m = torch.mean(float_x, [0, 2, 3], keepdim=True)
 m2 = torch.mean(float_x ** 2, [0, 2, 3], keepdim=True)
 var =(m2 - m ** 2)
 var = var.type(x.type())
 m = m.type(x.type())
 if return_mean_var:
return fused_bn(x, m, var, gain, bias, eps), m.squeeze() , var.squeeze()
else:
return fused_bn(x, m, var, gain, bias, eps)
 class myBN(nn.Module):
def __init__(self, num_channels, eps=1e-5, momentum=0.1):
super(myBN, self ).__init__()
self.momentum = momentum
self.eps = eps
self.momentum = momentum
self.register_buffer('stored_mean', torch.zeros(num_channels))
self.register_buffer('stored_var', torch.ones(num_channels))
self.register_buffer('accumulation_counter', torch.zeros(1))
self.accumulate_standing = False
def reset_stats(self):
self.stored_mean[:] = 0
self.stored_var[:] = 0
self.accumulation_counter[:] = 0
def forward(self, x, gain, bias):
if self.training:
out, mean, var = manual_bn(x, gain, bias, return_mean_var=True, eps=self.eps)
 if self.accumulate_standing:
self.stored_mean[:] = self.stored_mean + mean.data
self.stored_var[:] = self.stored_var + var.data
self.accumulation_counter += 1.0
else:
self.stored_mean[:] = self.stored_mean *(1 - self.momentum)+ mean * self.momentum
self.stored_var[:] = self.stored_var *(1 - self.momentum)+ var * self.momentum
return out
else:
mean = self.stored_mean.view(1, -1, 1, 1)
 var = self.stored_var.view(1, -1, 1, 1)
 if self.accumulate_standing:
mean = mean / self.accumulation_counter
var = var / self.accumulation_counter
return fused_bn(x, mean, var, gain, bias, self.eps)
 def groupnorm(x, norm_style):
if 'ch' in norm_style:
ch = int(norm_style.split('_')[-1])
 groups = max(int(x.shape[1])// ch, 1)
 elif 'grp' in norm_style:
groups = int(norm_style.split('_')[-1])
 else:
groups = 16
return F.group_norm(x, groups)
 class ccbn(nn.Module):
def __init__(self, output_size, input_size, which_linear, eps=1e-5, momentum=0.1,
cross_replica=False, mybn=False, norm_style='bn',):
super(ccbn, self ).__init__()
self.output_size, self.input_size = output_size, input_size
self.gain = which_linear(input_size, output_size)
 self.bias = which_linear(input_size, output_size)
 self.eps = eps
self.momentum = momentum
self.cross_replica = cross_replica
self.mybn = mybn
self.norm_style = norm_style
if self.mybn:
self.bn = myBN(output_size, self.eps, self.momentum)
 elif self.norm_style in ['bn', 'in']:
self.register_buffer('stored_mean', torch.zeros(output_size))
self.register_buffer('stored_var', torch.ones(output_size))
def forward(self, x, y):
gain =(1 + self.gain(y)).view(y.size(0), -1, 1, 1)
 bias = self.bias(y ).view(y.size(0), -1, 1, 1)
 if self.mybn:
return self.bn(x, gain=gain, bias=bias)
 else:
if self.norm_style == 'bn':
out = F.batch_norm(x, self.stored_mean, self.stored_var, None, None,
self.training, 0.1, self.eps)
 elif self.norm_style == 'in':
out = F.instance_norm(x, self.stored_mean, self.stored_var, None, None,
self.training, 0.1, self.eps)
 elif self.norm_style == 'gn':
out = groupnorm(x, self.normstyle)
 elif self.norm_style == 'nonorm':
out = x
return out * gain + bias
def extra_repr(self):
s = 'out: {output_size}, in: {input_size},'
s += ' cross_replica={cross_replica}'
return s.format(**self.__dict__)
 class bn(nn.Module):
def __init__(self, output_size, eps=1e-5, momentum=0.1,
cross_replica=False, mybn=False):
super(bn, self ).__init__()
self.output_size = output_size
self.gain = P(torch.ones(output_size), requires_grad=True)
 self.bias = P(torch.zeros(output_size), requires_grad=True)
 self.eps = eps
self.momentum = momentum
self.cross_replica = cross_replica
self.mybn = mybn
if mybn:
self.bn = myBN(output_size, self.eps, self.momentum)
 else:
self.register_buffer('stored_mean', torch.zeros(output_size))
self.register_buffer('stored_var', torch.ones(output_size))
def forward(self, x, y=None):
if self.mybn:
gain = self.gain.view(1, -1, 1, 1)
 bias = self.bias.view(1, -1, 1, 1)
 return self.bn(x, gain=gain, bias=bias)
 else:
return F.batch_norm(x, self.stored_mean, self.stored_var, self.gain,
self.bias, self.training, self.momentum, self.eps)
 class GBlock(nn.Module):
def __init__(self, in_channels, out_channels,
which_conv=nn.Conv2d, which_bn=bn, activation=None,
upsample=None):
super(GBlock, self ).__init__()
self.in_channels, self.out_channels = in_channels, out_channels
self.which_conv, self.which_bn = which_conv, which_bn
self.activation = activation
self.upsample = upsample
self.conv1 = self.which_conv(self.in_channels, self.out_channels)
 self.conv2 = self.which_conv(self.out_channels, self.out_channels)
 self.learnable_sc = in_channels != out_channels or upsample
if self.learnable_sc:
self.conv_sc = self.which_conv(in_channels, out_channels,
kernel_size=1, padding=0)
 self.bn1 = self.which_bn(in_channels)
 self.bn2 = self.which_bn(out_channels)
 self.upsample = upsample
def forward(self, x, y):
h = self.activation(self.bn1(x, y))
if self.upsample:
h = self.upsample(h)
 x = self.upsample(x)
 h = self.conv1(h)
 h = self.activation(self.bn2(h, y))
h = self.conv2(h)
 if self.learnable_sc:
x = self.conv_sc(x)
 return h + x
class DBlock(nn.Module):
def __init__(self, in_channels, out_channels, which_conv=SNConv2d, wide=True,
preactivation=False, activation=None, downsample=None,):
super(DBlock, self ).__init__()
self.in_channels, self.out_channels = in_channels, out_channels
self.hidden_channels = self.out_channels if wide else self.in_channels
self.which_conv = which_conv
self.preactivation = preactivation
self.activation = activation
self.downsample = downsample
self.conv1 = self.which_conv(self.in_channels, self.hidden_channels)
 self.conv2 = self.which_conv(self.hidden_channels, self.out_channels)
 self.learnable_sc = True if(in_channels != out_channels)or downsample else False
if self.learnable_sc:
self.conv_sc = self.which_conv(in_channels, out_channels,
kernel_size=1, padding=0)
 def shortcut(self, x):
if self.preactivation:
if self.learnable_sc:
x = self.conv_sc(x)
 if self.downsample:
x = self.downsample(x)
 else:
if self.downsample:
x = self.downsample(x)
 if self.learnable_sc:
x = self.conv_sc(x)
 return x
def forward(self, x):
if self.preactivation:
h = F.relu(x)
 else:
h = x
h = self.conv1(h)
 h = self.conv2(self.activation(h))
if self.downsample:
h = self.downsample(h)
 return h + self.shortcut(x )<init_hyperparams>
list_of_crew_names = train['crew'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values
Counter([i for j in list_of_crew_names for i in j] ).most_common(15 )
def G_arch(ch=64, attention='64', ksize='333333', dilation='111111'):
arch = {}
arch[512] = {'in_channels': [ch * item for item in [16, 16, 8, 8, 4, 2, 1]],
'out_channels': [ch * item for item in [16, 8, 8, 4, 2, 1, 1]],
'upsample': [True] * 7,
'resolution': [8, 16, 32, 64, 128, 256, 512],
'attention': {2 ** i:(2 ** i in [int(item)for item in attention.split('_')])
 for i in range(3, 10)}}
arch[256] = {'in_channels': [ch * item for item in [16, 16, 8, 8, 4, 2]],
'out_channels': [ch * item for item in [16, 8, 8, 4, 2, 1]],
'upsample': [True] * 6,
'resolution': [8, 16, 32, 64, 128, 256],
'attention': {2 ** i:(2 ** i in [int(item)for item in attention.split('_')])
 for i in range(3, 9)}}
arch[128] = {'in_channels': [ch * item for item in [16, 16, 8, 4, 2]],
'out_channels': [ch * item for item in [16, 8, 4, 2, 1]],
'upsample': [True] * 5,
'resolution': [8, 16, 32, 64, 128],
'attention': {2 ** i:(2 ** i in [int(item)for item in attention.split('_')])
 for i in range(3, 8)}}
arch[64] = {'in_channels': [ch * item for item in [16, 16, 8, 4]],
'out_channels': [ch * item for item in [16, 8, 4, 2]],
'upsample': [True] * 4,
'resolution': [8, 16, 32, 64],
'attention': {2 ** i:(2 ** i in [int(item)for item in attention.split('_')])
 for i in range(3, 7)}}
arch[32] = {'in_channels': [ch * item for item in [4, 4, 4]],
'out_channels': [ch * item for item in [4, 4, 4]],
'upsample': [True] * 3,
'resolution': [8, 16, 32],
'attention': {2 ** i:(2 ** i in [int(item)for item in attention.split('_')])
 for i in range(3, 6)}}
return arch
class Generator(nn.Module):
def __init__(self, G_ch=64, dim_z=128, bottom_width=4, resolution=128,
G_kernel_size=3, G_attn='64', n_classes=1000,
num_G_SVs=1, num_G_SV_itrs=1,
G_shared=True, shared_dim=0, hier=False,
cross_replica=False, mybn=False,
G_activation=nn.ReLU(inplace=False),
G_lr=5e-5, G_B1=0.0, G_B2=0.999, adam_eps=1e-8,
BN_eps=1e-5, SN_eps=1e-12, G_mixed_precision=False, G_fp16=False,
G_init='ortho', skip_init=False, no_optim=False,
G_param='SN', norm_style='bn',
**kwargs):
super(Generator, self ).__init__()
self.ch = G_ch
self.dim_z = dim_z
self.bottom_width = bottom_width
self.resolution = resolution
self.kernel_size = G_kernel_size
self.attention = G_attn
self.n_classes = n_classes
self.G_shared = G_shared
self.shared_dim = shared_dim if shared_dim > 0 else dim_z
self.hier = hier
self.cross_replica = cross_replica
self.mybn = mybn
self.activation = G_activation
self.init = G_init
self.G_param = G_param
self.norm_style = norm_style
self.BN_eps = BN_eps
self.SN_eps = SN_eps
self.fp16 = G_fp16
self.arch = G_arch(self.ch, self.attention)[resolution]
if self.hier:
self.num_slots = len(self.arch['in_channels'])+ 1
self.z_chunk_size =(self.dim_z // self.num_slots)
 self.dim_z = self.z_chunk_size * self.num_slots
else:
self.num_slots = 1
self.z_chunk_size = 0
if self.G_param == 'SN':
self.which_conv = partial(SNConv2d,
kernel_size=3, padding=1,
num_svs=num_G_SVs, num_itrs=num_G_SV_itrs,
eps=self.SN_eps)
 self.which_linear = partial(SNLinear,
num_svs=num_G_SVs, num_itrs=num_G_SV_itrs,
eps=self.SN_eps)
 else:
self.which_conv = partial(nn.Conv2d, kernel_size=3, padding=1)
 self.which_linear = nn.Linear
self.which_embedding = nn.Embedding
bn_linear =(partial(self.which_linear, bias=False)if self.G_shared else self.which_embedding)
 self.which_bn = partial(ccbn,
which_linear=bn_linear,
cross_replica=self.cross_replica,
mybn=self.mybn,
input_size=(self.shared_dim + self.z_chunk_size if self.G_shared else self.n_classes),
norm_style=self.norm_style,
eps=self.BN_eps)
 self.shared =(self.which_embedding(n_classes, self.shared_dim)if G_shared else identity())
 self.linear = self.which_linear(
 self.dim_z // self.num_slots,
self.arch['in_channels'][0] *(self.bottom_width ** 2))
self.blocks = []
for index in range(len(self.arch['out_channels'])) :
self.blocks += [[GBlock(
 in_channels=self.arch['in_channels'][index],
out_channels=self.arch['out_channels'][index],
which_conv=self.which_conv,
which_bn=self.which_bn,
activation=self.activation,
upsample=(partial(F.interpolate, scale_factor=2)if self.arch['upsample'][index] else None)
)]]
if self.arch['attention'][self.arch['resolution'][index]]:
print('Adding attention layer in G at resolution %d' % self.arch['resolution'][index])
 self.blocks[-1] += [Attention(self.arch['out_channels'][index], self.which_conv)]
self.blocks = nn.ModuleList([nn.ModuleList(block)for block in self.blocks])
 self.output_layer = nn.Sequential(
 bn(self.arch['out_channels'][-1], cross_replica=self.cross_replica, mybn=self.mybn),
self.activation,
self.which_conv(self.arch['out_channels'][-1], 3)
)
 if not skip_init:
self.init_weights()
if no_optim:
return
self.lr, self.B1, self.B2, self.adam_eps = G_lr, G_B1, G_B2, adam_eps
if G_mixed_precision:
print('Using fp16 adam in G...')
 self.optim = Adam16(params=self.parameters() , lr=self.lr,
betas=(self.B1, self.B2), weight_decay=0,
eps=self.adam_eps)
 else:
self.optim = optim.Adam(params=self.parameters() , lr=self.lr,
betas=(self.B1, self.B2), weight_decay=0,
eps=self.adam_eps)
 def init_weights(self):
self.param_count = 0
for module in self.modules() :
if(isinstance(module, nn.Conv2d)
 or isinstance(module, nn.Linear)
 or isinstance(module, nn.Embedding)) :
if self.init == 'ortho':
init.orthogonal_(module.weight)
 elif self.init == 'N02':
init.normal_(module.weight, 0, 0.02)
 elif self.init in ['glorot', 'xavier']:
init.xavier_uniform_(module.weight)
 else:
print('Init style not recognized...')
 self.param_count += sum([p.data.nelement() for p in module.parameters() ])
 print('Param count for G''s initialized parameters: %d' % self.param_count)
 def forward(self, z, y):
if self.hier:
zs = torch.split(z, self.z_chunk_size, 1)
 z = zs[0]
ys = [torch.cat([y, item], 1)for item in zs[1:]]
else:
ys = [y] * len(self.blocks)
 h = self.linear(z)
 h = h.view(h.size(0), -1, self.bottom_width, self.bottom_width)
 for index, blocklist in enumerate(self.blocks):
for block in blocklist:
h = block(h, ys[index])
 return torch.tanh(self.output_layer(h))
def D_arch(ch=64, attention='64', ksize='333333', dilation='111111'):
arch = {}
arch[256] = {'in_channels': [3] + [ch * item for item in [1, 2, 4, 8, 8, 16]],
'out_channels': [item * ch for item in [1, 2, 4, 8, 8, 16, 16]],
'downsample': [True] * 6 + [False],
'resolution': [128, 64, 32, 16, 8, 4, 4],
'attention': {2 ** i: 2 ** i in [int(item)for item in attention.split('_')]
for i in range(2, 8)}}
arch[128] = {'in_channels': [3] + [ch * item for item in [1, 2, 4, 8, 16]],
'out_channels': [item * ch for item in [1, 2, 4, 8, 16, 16]],
'downsample': [True] * 5 + [False],
'resolution': [64, 32, 16, 8, 4, 4],
'attention': {2 ** i: 2 ** i in [int(item)for item in attention.split('_')]
for i in range(2, 8)}}
arch[64] = {'in_channels': [3] + [ch * item for item in [1, 2, 4, 8]],
'out_channels': [item * ch for item in [1, 2, 4, 8, 16]],
'downsample': [True] * 4 + [False],
'resolution': [32, 16, 8, 4, 4],
'attention': {2 ** i: 2 ** i in [int(item)for item in attention.split('_')]
for i in range(2, 7)}}
arch[32] = {'in_channels': [3] + [item * ch for item in [4, 4, 4]],
'out_channels': [item * ch for item in [4, 4, 4, 4]],
'downsample': [True, True, False, False],
'resolution': [16, 16, 16, 16],
'attention': {2 ** i: 2 ** i in [int(item)for item in attention.split('_')]
for i in range(2, 6)}}
return arch
class Discriminator(nn.Module):
def __init__(self, D_ch=64, D_wide=True, resolution=128,
D_kernel_size=3, D_attn='64', n_classes=1000,
num_D_SVs=1, num_D_SV_itrs=1, D_activation=nn.ReLU(inplace=False),
D_lr=2e-4, D_B1=0.0, D_B2=0.999, adam_eps=1e-8,
SN_eps=1e-12, output_dim=1, D_mixed_precision=False, D_fp16=False,
D_init='ortho', skip_init=False, D_param='SN', **kwargs):
super(Discriminator, self ).__init__()
self.ch = D_ch
self.D_wide = D_wide
self.resolution = resolution
self.kernel_size = D_kernel_size
self.attention = D_attn
self.n_classes = n_classes
self.activation = D_activation
self.init = D_init
self.D_param = D_param
self.SN_eps = SN_eps
self.fp16 = D_fp16
self.arch = D_arch(self.ch, self.attention)[resolution]
if self.D_param == 'SN':
self.which_conv = partial(SNConv2d,
kernel_size=3, padding=1,
num_svs=num_D_SVs, num_itrs=num_D_SV_itrs,
eps=self.SN_eps)
 self.which_linear = partial(SNLinear,
num_svs=num_D_SVs, num_itrs=num_D_SV_itrs,
eps=self.SN_eps)
 self.which_embedding = partial(SNEmbedding,
num_svs=num_D_SVs, num_itrs=num_D_SV_itrs,
eps=self.SN_eps)
 self.blocks = []
for index in range(len(self.arch['out_channels'])) :
self.blocks += [[DBlock(
 in_channels=self.arch['in_channels'][index],
out_channels=self.arch['out_channels'][index],
which_conv=self.which_conv,
wide=self.D_wide,
activation=self.activation,
preactivation=(index > 0),
downsample=(nn.AvgPool2d(2)if self.arch['downsample'][index] else None)
)]]
if self.arch['attention'][self.arch['resolution'][index]]:
print('Adding attention layer in D at resolution %d' % self.arch['resolution'][index])
 self.blocks[-1] += [Attention(self.arch['out_channels'][index], self.which_conv)]
self.blocks = nn.ModuleList([nn.ModuleList(block)for block in self.blocks])
 self.linear = self.which_linear(self.arch['out_channels'][-1], output_dim)
 self.embed = self.which_embedding(self.n_classes, self.arch['out_channels'][-1])
 if not skip_init:
self.init_weights()
self.lr, self.B1, self.B2, self.adam_eps = D_lr, D_B1, D_B2, adam_eps
if D_mixed_precision:
print('Using fp16 adam in D...')
 self.optim = Adam16(params=self.parameters() , lr=self.lr,
betas=(self.B1, self.B2), weight_decay=0, eps=self.adam_eps)
 else:
self.optim = optim.Adam(params=self.parameters() , lr=self.lr,
betas=(self.B1, self.B2), weight_decay=0, eps=self.adam_eps)
 def init_weights(self):
self.param_count = 0
for module in self.modules() :
if(isinstance(module, nn.Conv2d)
 or isinstance(module, nn.Linear)
 or isinstance(module, nn.Embedding)) :
if self.init == 'ortho':
init.orthogonal_(module.weight)
 elif self.init == 'N02':
init.normal_(module.weight, 0, 0.02)
 elif self.init in ['glorot', 'xavier']:
init.xavier_uniform_(module.weight)
 else:
print('Init style not recognized...')
 self.param_count += sum([p.data.nelement() for p in module.parameters() ])
 print('Param count for D''s initialized parameters: %d' % self.param_count)
 def forward(self, x, y=None):
h = x
for index, blocklist in enumerate(self.blocks):
for block in blocklist:
h = block(h)
 h = torch.sum(self.activation(h), [2, 3])
 out = self.linear(h)
 out = out + torch.sum(self.embed(y)* h, 1, keepdim=True)
 return out
class G_D(nn.Module):
def __init__(self, G, D):
super(G_D, self ).__init__()
self.G = G
self.D = D
def forward(self, z, gy, x=None, dy=None, train_G=False, return_G_z=False,
split_D=False):
with torch.set_grad_enabled(train_G):
G_z = self.G(z, self.G.shared(gy))
if self.G.fp16 and not self.D.fp16:
G_z = G_z.float()
if self.D.fp16 and not self.G.fp16:
G_z = G_z.half()
if split_D:
D_fake = self.D(G_z, gy)
 if x is not None:
D_real = self.D(x, dy)
 return D_fake, D_real
else:
if return_G_z:
return D_fake, G_z
else:
return D_fake
else:
D_input = torch.cat([G_z, x], 0)if x is not None else G_z
D_class = torch.cat([gy, dy], 0)if dy is not None else gy
D_out = self.D(D_input, D_class)
 if x is not None:
return torch.split(D_out, [G_z.shape[0], x.shape[0]])
 else:
if return_G_z:
return D_out, G_z
else:
return D_out<normalization>
train['num_crew'] = train['crew'].apply(lambda x: len(x)if x != {} else 0)
 top_crew_names = [m[0] for m in Counter([i for j in list_of_crew_names for i in j] ).most_common(15)]
for g in top_crew_names:
train['crew_name_' + g] = train['crew'].apply(lambda x: 1 if g in str(x)else 0)
 train['genders_0_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['genders_1_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['genders_2_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
top_crew_jobs = [m[0] for m in Counter([i for j in list_of_crew_jobs for i in j] ).most_common(15)]
for j in top_crew_jobs:
train['jobs_' + j] = train['crew'].apply(lambda x: sum([1 for i in x if i['job'] == j]))
top_crew_departments = [m[0] for m in Counter([i for j in list_of_crew_departments for i in j] ).most_common(15)]
for j in top_crew_departments:
train['departments_' + j] = train['crew'].apply(lambda x: sum([1 for i in x if i['department'] == j]))
test['num_crew'] = test['crew'].apply(lambda x: len(x)if x != {} else 0)
 for g in top_crew_names:
test['crew_name_' + g] = test['crew'].apply(lambda x: 1 if g in str(x)else 0)
 test['genders_0_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['genders_1_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['genders_2_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
for j in top_crew_jobs:
test['jobs_' + j] = test['crew'].apply(lambda x: sum([1 for i in x if i['job'] == j]))
for j in top_crew_departments:
test['departments_' + j] = test['crew'].apply(lambda x: sum([1 for i in x if i['department'] == j]))
train = train.drop(['crew'], axis=1)
 test = test.drop(['crew'], axis=1 )
class MyImg() :
def __init__(self, img, tfm):
self.px = np.array(img)
 self.tfm = tfm
@property
def size(self):
h, w, _ = self.px.shape
return min(w, h)
 def pad(img, padding_mode='reflect'):
p = math.ceil(( max(img.size)- min(img.size)) / 2)
 p_horr = p if img.width < img.height else 0
p_vert = p if img.height < img.width else 0
img = T.Pad(( p_horr, p_vert), padding_mode=padding_mode )(img)
 if img.width != img.height:
s = min(img.size)
 img = img.crop(( 0, 0, s, s))
return img
def take_top(img):
size = min(img.size)
 bbox =(0, 0, size, size)
 return img.crop(bbox)
 def take_diagonal(img):
w, h = img.size
size = min(w, h)
 bbox_l =(0, 0, size, size)
 bbox_r =(w - size, h - size, w, h)
 return [img.crop(bbox_l), img.crop(bbox_r)]
resize = T.Resize(IMG_SIZE, interpolation=Image.LANCZOS)
 resize2x = T.Resize(IMG_SIZE_2, interpolation=Image.LANCZOS)
 top_crop = T.Compose([T.Lambda(take_top), resize])
 top_crop2x = T.Compose([T.Lambda(take_top), resize2x])
 two_crops = T.Compose([resize, T.Lambda(take_diagonal)])
 two_crops2x = T.Compose([resize2x, T.Lambda(take_diagonal)])
 pad_only = T.Compose([T.Lambda(pad), resize])
 pad_only2x = T.Compose([T.Lambda(pad), resize2x])
 @jit(nopython=True)
 def pad_one_dim(clow, chigh, pad, cmax):
clow = max(0, clow - pad)
 chigh = min(cmax, chigh + pad)
 return clow, chigh, chigh - clow
def calc_bbox(obj, img_w, img_h, zoom=0.0, try_square=True):
bndbox = obj.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 xmin, xmax, obj_w = pad_one_dim(xmin, xmax, 0, img_w)
 ymin, ymax, obj_h = pad_one_dim(ymin, ymax, 0, img_h)
 if zoom != 0.0:
pad_w = obj_w * zoom / 2
pad_h = obj_h * zoom / 2
xmin, xmax, obj_w = pad_one_dim(xmin, xmax, pad_w, img_w)
 ymin, ymax, obj_h = pad_one_dim(ymin, ymax, pad_h, img_h)
 if try_square:
if obj_w > obj_h:
pad =(obj_w - obj_h)/ 2
ymin, ymax, obj_h = pad_one_dim(ymin, ymax, pad, img_h)
 elif obj_h > obj_w:
pad =(obj_h - obj_w)/ 2
xmin, xmax, obj_w = pad_one_dim(xmin, xmax, pad, img_w)
 if obj_w > obj_h:
pad = obj_w - obj_h
ymin, ymax, obj_h = pad_one_dim(ymin, ymax, pad, img_h)
 elif obj_h > obj_w:
pad = obj_h - obj_w
xmin, xmax, obj_w = pad_one_dim(xmin, xmax, pad, img_w)
 return int(xmin), int(ymin), int(xmax), int(ymax)
 @jit(nopython=True)
 def bb2wh(bbox):
width = bbox[2] - bbox[0]
height = bbox[3] - bbox[1]
return width, height
def create_counters() :
return { 'top': 0, 'sides': 0, 'slide': 0, 'pad': 0, 'skip': 0, 'noop': 0 }
def make_x1res(img, bbox, counter):
ar = img.width / img.height
if ar == 1.0:
tfm_imgs = resize(img); counter['noop'] += 1
elif 1.0 < ar < 1.4 or 1.0 < 1/ar < 1.25:
tfm_imgs = pad_only(img); counter['pad'] += 1
elif 1.4 <= ar < 2.0:
tfm_imgs = two_crops(img); counter['sides'] += 2
elif 1.25 <= 1/ar < 1.5:
tfm_imgs = [pad_only(img), top_crop(img)]
counter['pad'] += 1; counter['top'] += 1
elif 1.5 <= 1/ar < 1.8:
tfm_imgs = top_crop(img); counter['top'] += 1
else:
tfm_imgs = None; counter['skip'] += 1
return tfm_imgs
def make_x2res(img, bbox, counter):
ar = img.width / img.height
if ar == 1.0:
tfm_imgs = resize2x(img); counter['noop'] += 1
elif 1.0 < ar < 1.4 or 1.0 < 1/ar < 1.4:
tfm_imgs = pad_only2x(img); counter['pad'] += 1
else:
tfm_imgs = None
return tfm_imgs
def add_sample(samples, label, tfm, imgs, labels):
if not samples: return
elif isinstance(samples, Image.Image):
imgs.append(MyImg(samples, tfm))
labels.append(label)
 elif isinstance(samples, list):
imgs.extend([MyImg(s, tfm)for s in samples])
 labels.extend([label] * len(samples))
else: assert False
def is_valid_file(x):
return datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 class DogsDataSet(datasets.vision.VisionDataset):
def __init__(self, root, transforms, target_transform=None, max_samples=None):
super().__init__(root, transform=None)
 assert isinstance(transforms, list)and len(transforms)== 4
self.transforms = transforms
self.target_transform = target_transform
self.max_samples = max_samples
self.classes = {}
imgs, labels = self._load_subfolders_images(self.root)
 assert len(imgs)== len(labels)
 if len(imgs)== 0:
raise RuntimeError(f'Found 0 files in subfolders of: {self.root}')
 self.imgs = imgs
self.labels = labels
def _create_or_get_class(self, name):
try:
label = self.classes[name]
except KeyError:
label = len(self.classes)
 self.classes[name] = label
return label
def _load_subfolders_images(self, root):
imgs, labels, paths = [], [], []
add_sample_ = partial(add_sample, imgs=imgs, labels=labels)
 low_res = lambda bbox: min(bb2wh(bbox)) < IMG_SIZE
can_2x = lambda bbox: min(bb2wh(bbox)) >= IMG_SIZE_2
counter = create_counters()
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames):
path = os.path.join(root, fname)
 paths.append(path)
 if self.max_samples:
paths = paths[:self.max_samples]
for path in paths:
if not is_valid_file(path): continue
img = datasets.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(dirname for dirname in os.listdir(ANNOT_PATH)if dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join(ANNOT_PATH, annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
name = o.find('name' ).text
label = self._create_or_get_class(name)
 prev_bbox, tfm_imgs = None, None
bbox = calc_bbox(o, img_w=img.width, img_h=img.height, zoom=0.03125)
 if low_res(bbox):
counter['skip'] += 1; continue
obj_img = img.crop(bbox)
 ar = obj_img.width / obj_img.height
tfm_imgs = make_x1res(obj_img, bbox, counter)
 add_sample_(tfm_imgs, label, 0)
 if can_2x(bbox):
tfm_imgs = make_x2res(obj_img, bbox, counter)
 add_sample_(tfm_imgs, label, 1)
 prev_bbox = bbox
bbox = calc_bbox(o, img_w=img.width, img_h=img.height, zoom=0.1)
 if not(bbox == prev_bbox or low_res(bbox)) :
obj_img = img.crop(bbox)
 ar = obj_img.width / obj_img.height
if 1.05 < ar < 1.25:
tfm_imgs = resize(obj_img); counter['slide'] += 1
add_sample_(tfm_imgs, label, 3)
 if can_2x(bbox):
tfm_imgs = make_x2res(obj_img, bbox, counter)
 add_sample_(tfm_imgs, label, 2)
 bbox = calc_bbox(o, img_w=img.width, img_h=img.height, zoom=0.12, try_square=False)
 if not low_res(bbox):
obj_img = img.crop(bbox)
 ar = obj_img.width / obj_img.height
if 1.3 <= 1/ar < 1.8:
tfm_imgs = top_crop(obj_img); counter['top'] += 1
add_sample_(tfm_imgs, label, 0)
 if can_2x(bbox):
tfm_imgs = make_x2res(obj_img, bbox, counter)
 add_sample_(tfm_imgs, label, 1)
 n_x1, n_x2 = 0, 0
for i, img in enumerate(imgs):
if img.size == IMG_SIZE:
n_x1 +=1
else:
n_x2 +=1
print(f'Found {len(self.classes)} classes
Loaded 64x64 {n_x1} images
'
f'Loaded 128x128 {n_x2} images
')
 print(counter)
 return imgs, labels
def __getitem__(self, index):
img = self.imgs[index]
label = self.labels[index]
tfms = self.transforms[img.tfm]
img = tfms(image=img.px)['image']
if self.target_transform:
label = self.target_transform(label)
 return img, label
def __len__(self):
return len(self.imgs)
 def create_runtime_tfms() :
mean, std = [0.5]*3, [0.5]*3
resize_to_64 = A.SmallestMaxSize(IMG_SIZE, interpolation=cv2.INTER_LANCZOS4)
 out = [A.HorizontalFlip(p=0.5), A.Normalize(mean=mean, std=std), ToTensor() ]
rand_crop = A.RandomCrop(IMG_SIZE, IMG_SIZE)
 affine_1 = A.ShiftScaleRotate(
 shift_limit=0, scale_limit=0.1, rotate_limit=8,
interpolation=cv2.INTER_CUBIC,
border_mode=cv2.BORDER_REFLECT_101, p=1.0)
 affine_1 = A.Compose([affine_1, resize_to_64])
 affine_2 = A.ShiftScaleRotate(
 shift_limit=0.06, scale_limit=(-0.06, 0.14), rotate_limit=6,
interpolation=cv2.INTER_CUBIC,
border_mode=cv2.BORDER_REFLECT_101, p=1.0)
 affine_2 = A.Compose([affine_2, resize_to_64])
 tfm_0 = A.Compose(out)
 tfm_1 = A.Compose([affine_1, *out])
 tfm_2 = A.Compose([affine_2, *out])
 tfm_3 = A.Compose([rand_crop, *out])
 return [tfm_0, tfm_1, tfm_2, tfm_3]
def get_data_loaders(data_root=None, batch_size=32, num_workers=2, shuffle=True,
pin_memory=True, drop_last=True):
print('Using dataset root location %s' % data_root)
 train_set = DogsDataSet(data_root, create_runtime_tfms())
 loaders = []
loader_kwargs = {'num_workers': num_workers, 'pin_memory': pin_memory,
'drop_last': drop_last}
train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=shuffle, **loader_kwargs)
 loaders.append(train_loader)
 return loaders<categorify>
train['log_revenue'] = np.log1p(train['revenue'] )
def seed_rng(seed):
torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 np.random.seed(seed)
 def update_config_roots(config):
if config['base_root']:
print('Pegging all root folders to base root %s' % config['base_root'])
 for key in ['data', 'weights', 'logs', 'samples']:
config['%s_root' % key] = '%s/%s' %(config['base_root'], key)
 return config
def prepare_root(config):
for key in ['weights_root', 'logs_root', 'samples_root']:
if not os.path.exists(config[key]):
print('Making directory %s for %s...' %(config[key], key))
os.mkdir(config[key])
 def join_strings(base_string, strings):
return base_string.join([item for item in strings if item] )<prepare_x_and_y>
train['log_budget'] = np.log1p(train['budget'])
 test['log_budget'] = np.log1p(test['budget'] )
def run(G, D, GD, loaders, state_dict, config, device, G_ema=None, ema=None):
G_batch_size = max(config['G_batch_size'], config['batch_size'])
 z_, y_ = prepare_z_y(
 G_batch_size, G.dim_z, config['n_classes'], device=device, fp16=config['G_fp16'])
 fixed_z, fixed_y = prepare_z_y(
 G_batch_size, G.dim_z, config['n_classes'], device=device, fp16=config['G_fp16'])
 fixed_z.sample_()
fixed_y.sample_()
train = create_train_fn(G, D, GD, z_, y_, ema, state_dict, config)
 print('Beginning training at epoch %d...' % state_dict['epoch'])
 start_time = time.perf_counter()
total_iters = config['num_epochs'] * len(loaders[0])
 for epoch in range(state_dict['epoch'], config['num_epochs']):
for i,(x, y)in enumerate(loaders[0]):
state_dict['itr'] += 1
G.train()
D.train()
if config['ema']:
G_ema.train()
if config['D_fp16']:
x, y = x.to(device ).half() , y.to(device)
 else:
x, y = x.to(device), y.to(device)
 metrics = train(x, y)
 if not(state_dict['itr'] % config['log_interval']):
curr_time = time.perf_counter()
curr_time_str = datetime.datetime.fromtimestamp(curr_time ).strftime('%H:%M:%S')
 elapsed = str(datetime.timedelta(seconds=(curr_time - start_time)))
 log =(
 "[{}] [{}] [{} / {}] Ep {}, ".format(curr_time_str, elapsed, state_dict['itr'], total_iters, epoch)+
', '.join(['%s : %+4.3f' %(key, metrics[key])for key in metrics])
)
 print(log)
 if not(state_dict['itr'] % config['save_every']):
if config['G_eval_mode']:
print('Switchin G to eval mode...')
 G.eval()
save_and_sample(G, G_ema, fixed_z, fixed_y, config)
 if config['stop_after'] > 0 and int(time.perf_counter() - start_time)> config['stop_after']:
print("Time limit reached! Stopping training!")
 return
state_dict['epoch'] += 1<split>
train['homepage'].value_counts().head()
def trunc_trick(bs, z_dim, bound=2.0):
z = torch.randn(bs, z_dim)
 while z.min() < -bound or bound < z.max() :
z = z.where(( -bound < z)&(z < bound), torch.randn_like(z))
return z
def collect_bn_stats(G, n_samples, config):
im_batch_size = config['n_classes']
G.train()
for i_batch in range(0, n_samples, im_batch_size):
with torch.no_grad() :
z = torch.randn(im_batch_size, G.dim_z, device=device)
 y = torch.arange(im_batch_size ).to(device)
 images = G(z, G.shared(y)).float().cpu()
def generate_images(out_dir, G, n_images, config):
im_batch_size = config['n_classes']
z_bound = config['trunc_z']
if z_bound > 0.0: print(f'Truncating z to(-{z_bound}, {z_bound})')
 for i_batch in range(0, n_images, im_batch_size):
with torch.no_grad() :
if z_bound > 0.0:
z = trunc_trick(im_batch_size, G.dim_z, bound=z_bound ).to(device)
 else:
z = torch.randn(im_batch_size, G.dim_z, device=device)
 y = torch.arange(im_batch_size ).to(device)
 images = G(z, G.shared(y)).float().cpu()
if i_batch + im_batch_size > n_images:
n_last_images = n_images - i_batch
print(f'Taking only {n_last_images} images from the last batch...')
 images = images[:n_last_images]
for i_image, image in enumerate(images):
fname = os.path.join(out_dir, f'image_{i_batch+i_image:05d}.png')
 image = denorm(image)
 if config['denoise']:
image = image * 256
image = image.numpy().transpose(1, 2, 0 ).astype('uint8')
 image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
 image = cv2.fastNlMeansDenoisingColored(
 src=image, dst=None,
h=config['denoise_str_lum'],
hColor=config['denoise_str_col'],
templateWindowSize=config['denoise_kernel_size'],
searchWindowSize=config['denoise_search_window'] 
)
 cv2.imwrite(fname, image)
 else:
torchvision.utils.save_image(image, fname)
 def show_saved_samples(n_images=256, cols=8, size=3):
rows = math.ceil(n_images / cols)
 _, axes = plt.subplots(rows, cols, figsize=(cols * size, rows * size))
for i, ax in enumerate(axes.flatten()):
img = Image.open(f'{OUT_DIR}/image_{i:05d}.png')
 ax.get_xaxis().set_visible(False)
 ax.get_yaxis().set_visible(False)
 ax.imshow(img)
 plt.subplots_adjust(wspace=0.0, hspace=0.0)
 plt.show()<choose_model_class>
train['has_homepage'] = 0
train.loc[train['homepage'].isnull() == False, 'has_homepage'] = 1
test['has_homepage'] = 0
test.loc[test['homepage'].isnull() == False, 'has_homepage'] = 1
activation_dict = {
'inplace_relu': nn.ReLU(inplace=True),
'relu': nn.ReLU(inplace=False),
'ir': nn.ReLU(inplace=True),
}
config = {
'num_workers': 2,
'pin_memory': True,
'shuffle': True,
'use_multiepoch_sampler': False,
'model': 'BigGAN',
'G_param': 'SN',
'D_param': 'SN',
'G_ch': 64,
'D_ch': 64,
'G_depth': 1,
'D_depth': 1,
'D_wide': True,
'G_shared': False,
'shared_dim': 0,
'dim_z': 128,
'z_var': 1.0,
'hier': False,
'cross_replica': False,
'mybn': False,
'G_nl': 'relu',
'D_nl': 'relu',
'G_attn': '64',
'D_attn': '64',
'norm_style': 'bn',
'seed': 0,
'G_init': 'ortho',
'D_init': 'ortho',
'skip_init': False,
'G_lr': 5e-5,
'D_lr': 2e-4,
'G_B1': 0.0,
'D_B1': 0.0,
'G_B2': 0.999,
'D_B2': 0.999,
'batch_size': 64,
'G_batch_size': 0,
'num_G_accumulations': 1,
'num_D_steps': 2,
'num_D_accumulations': 1,
'split_D': False,
'num_epochs': 100,
'parallel': False,
'G_fp16': False,
'D_fp16': False,
'D_mixed_precision': False,
'G_mixed_precision': False,
'accumulate_stats': False,
'num_standing_accumulations': 16,
'G_eval_mode': False,
'save_every': 2000,
'num_save_copies': 2,
'num_best_copies': 2,
'which_best': 'IS',
'no_fid': False,
'test_every': 5000,
'num_inception_images': 50000,
'hashname': False,
'base_root': '',
'data_root': '.. /data',
'weights_root': '.. /weights',
'logs_root': '.. /logs',
'samples_root': '.. /samples',
'pbar': 'mine',
'name_suffix': '',
'experiment_name': '',
'config_from_name': False,
'ema': False,
'ema_decay': 0.9999,
'use_ema': False,
'ema_start': 0,
'adam_eps': 1e-8,
'BN_eps': 1e-5,
'SN_eps': 1e-8,
'num_G_SVs': 1,
'num_D_SVs': 1,
'num_G_SV_itrs': 1,
'num_D_SV_itrs': 1,
'G_ortho': 0.0,
'D_ortho': 0.0,
'toggle_grads': True,
'which_train_fn': 'GAN',
'load_weights': '',
'resume': False,
'logstyle': '%3.3e',
'log_G_spectra': False,
'log_D_spectra': False,
'sv_log_interval': 10,
'log_interval': 100,
}
config['resolution'] = 64
config['n_classes'] = 120
config['data_root'] = INPUT_PATH<set_options>
vectorizer = TfidfVectorizer(
 sublinear_tf=True,
analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
min_df=5)
 overview_text = vectorizer.fit_transform(train['overview'].fillna(''))
linreg = LinearRegression()
linreg.fit(overview_text, train['log_revenue'])
 eli5.show_weights(linreg, vec=vectorizer, top=20, feature_filter=lambda x: x != '<BIAS>' )
config['num_epochs'] = 100
config['num_D_steps'] = 1
config['G_lr'] = 1e-4 * 0.85
config['D_lr'] = 4e-4 * 0.85
config['D_B2'] = 0.999
config['G_B2'] = 0.999
config['G_ch'] = 32
config['D_ch'] = 64
config['G_attn'] = '32'
config['D_attn'] = '32'
config['G_nl'] = 'inplace_relu'
config['D_nl'] = 'inplace_relu'
config['G_shared'] = True
config['shared_dim'] = 128
config['dim_z'] = 120
config['hier'] = True
config['SN_eps'] = 1e-8
config['BN_eps'] = 1e-5
config['adam_eps'] = 1e-8
config['G_ortho'] = 0.0
config['G_init'] = 'ortho'
config['D_init'] = 'ortho'
config['G_eval_mode'] = True
config['save_every'] = 5000
config['log_interval'] = 500
config['batch_size'] = 32
config['ema'] = True
config['use_ema'] = True
config['ema_decay'] = 0.9999
config['ema_start'] = 50_000
config['stop_after'] = 31_500
config['trunc_z'] = 0.0
config['denoise'] = True
config['denoise_str_lum'] = 2.65
config['denoise_str_col'] = 1.0
config['denoise_kernel_size'] = 7
config['denoise_search_window'] = 11<set_options>
test.loc[test['release_date'].isnull() == True, 'release_date'] = '01/01/98'
config = update_config_roots(config)
 device = 'cuda'
seed_rng(1234)
 prepare_root(config)
 torch.backends.cudnn.benchmark = True<define_variables>
def fix_date(x):

 year = x.split('/')[2]
if int(year)<= 19:
return x[:-2] + '20' + year
else:
return x[:-2] + '19' + year
%%time
D_batch_size =(config['batch_size'] * config['num_D_steps'] * config['num_D_accumulations'])
 loaders = get_data_loaders(
 data_root=INPUT_PATH,
batch_size=D_batch_size,
num_workers=config['num_workers'],
shuffle=config['shuffle'],
pin_memory=config['pin_memory'],
drop_last=True
)<define_search_model>
train['release_date'] = train['release_date'].apply(lambda x: fix_date(x))
test['release_date'] = test['release_date'].apply(lambda x: fix_date(x))
train['release_date'] = pd.to_datetime(train['release_date'])
 test['release_date'] = pd.to_datetime(test['release_date'] )
config['G_activation'] = activation_dict[config['G_nl']]
config['D_activation'] = activation_dict[config['D_nl']]
G = Generator(**config ).to(device)
 D = Discriminator(**config ).to(device)
 if config['ema']:
print('Preparing EMA for G with decay of {}'.format(config['ema_decay']))
G_ema = Generator(**{**config, 'skip_init': True, 'no_optim': True} ).to(device)
 ema = apply_ema(G, G_ema, config['ema_decay'], config['ema_start'])
 else:
G_ema, ema = None, None
GD = G_D(G, D)
 print('Number of params in G: {} D: {}'.format(
 *[sum([p.data.nelement() for p in net.parameters() ])for net in [G, D]]))<define_variables>
def process_date(df):
date_parts = ["year", "weekday", "month", 'weekofyear', 'day', 'quarter']
for part in date_parts:
part_col = 'release_date' + "_" + part
df[part_col] = getattr(df['release_date'].dt, part ).astype(int)
 return df
train = process_date(train)
 test = process_date(test )
state_dict = {
'itr': 0,
'epoch': 0,
'config': config
}<train_model>
train['status'].value_counts()
run(G, D, GD, loaders, state_dict, config, device, G_ema=G_ema, ema=ema )<set_options>
test['status'].value_counts()
%%time
if not os.path.exists(OUT_DIR):
os.mkdir(OUT_DIR)
 which_G = G_ema if config['ema'] and config['use_ema'] else G
if config['use_ema']:
collect_bn_stats(G_ema, 10_000, config)
 gc.collect()
G.eval()
generate_images(OUT_DIR, which_G, 10_000, config)
 shutil.make_archive('images', 'zip', OUT_DIR )<import_modules>
train = train.drop(['homepage', 'imdb_id', 'poster_path', 'release_date', 'status', 'log_revenue'], axis=1)
 test = test.drop(['homepage', 'imdb_id', 'poster_path', 'release_date', 'status'], axis=1 )
batch_size = 32<define_variables>
for col in train.columns:
if train[col].nunique() == 1:
print(col)
 train = train.drop([col], axis=1)
 test = test.drop([col], axis=1 )
start = time()<set_options>
for col in ['original_language', 'collection_name', 'all_genres']:
le = LabelEncoder()
le.fit(list(train[col].fillna('')) + list(test[col].fillna('')))
 train[col] = le.transform(train[col].fillna('' ).astype(str))
test[col] = le.transform(test[col].fillna('' ).astype(str))
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<load_pretrained>
train_texts = train[['title', 'tagline', 'overview', 'original_title']]
test_texts = test[['title', 'tagline', 'overview', 'original_title']]
class DataGenerator(Dataset):
def __init__(self, directory, transform=None, n_samples=np.inf):
self.directory = directory
self.transform = transform
self.n_samples = n_samples
self.samples = self._load_subfolders_images(directory)
 if len(self.samples)== 0:
raise RuntimeError("Found 0 files in subfolders of: {}".format(directory))
def _load_subfolders_images(self, root):
IMG_EXTENSIONS =(
 '.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
 aug_pipeline = iaa.Sequential([
iaa.ContrastNormalization(( 1.0, 1.3)) ,
iaa.Affine(scale=(1.0, 1.2)) ,
iaa.Sometimes(0.5, iaa.Sharpen(alpha=(0, 0.1), lightness=(1, 1.2))),
])
 def is_valid_file(x):
return torchvision.datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 required_transforms = torchvision.transforms.Compose([
torchvision.transforms.Resize(64),
torchvision.transforms.CenterCrop(64),
])
 imgs = []
paths = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames)[:min(self.n_samples, 999999999999999)]:
path = os.path.join(root, fname)
 paths.append(path)
 for path in tqdm(paths):
if is_valid_file(path):
img = dset.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(
 dirname for dirname in os.listdir('.. /input/annotation/Annotation/')if
dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join('.. /input/annotation/Annotation/',
annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
bbox =(xmin, ymin, xmin+w, ymin+w)
 object_img = required_transforms(img.crop(bbox))
imgs.append(object_img)
 if np.random.randint(0, 2)== 0:
aug_img = Image.fromarray(aug_pipeline.augment_image(np.array(object_img)))
 imgs.append(aug_img)
 return imgs
def __getitem__(self, index):
sample = self.samples[index]
if self.transform is not None:
sample = self.transform(sample)
 return np.asarray(sample)
 def __len__(self):
return len(self.samples)
<categorify>
for col in ['title', 'tagline', 'overview', 'original_title']:
train['len_' + col] = train[col].fillna('' ).apply(lambda x: len(str(x)))
 train['words_' + col] = train[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
train = train.drop(col, axis=1)
 test['len_' + col] = test[col].fillna('' ).apply(lambda x: len(str(x)))
 test['words_' + col] = test[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
test = test.drop(col, axis=1 )
%%time
database = '.. /input/all-dogs/all-dogs/'
transform = transforms.Compose([transforms.RandomRotation(5),
transforms.RandomHorizontalFlip(p=0.2),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 train_data = DataGenerator(database, transform=transform,n_samples=25000)
 train_loader = torch.utils.data.DataLoader(train_data, shuffle=True,batch_size=batch_size, num_workers = 4)
 print(len(train_loader))<normalization>
train.loc[train['id'] == 16,'revenue'] = 192864
train.loc[train['id'] == 90,'budget'] = 30000000
train.loc[train['id'] == 118,'budget'] = 60000000
train.loc[train['id'] == 149,'budget'] = 18000000
train.loc[train['id'] == 313,'revenue'] = 12000000
train.loc[train['id'] == 451,'revenue'] = 12000000
train.loc[train['id'] == 464,'budget'] = 20000000
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 930000
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 819,'budget'] = 90000000
train.loc[train['id'] == 850,'budget'] = 90000000
train.loc[train['id'] == 1112,'budget'] = 7500000
train.loc[train['id'] == 1131,'budget'] = 4300000
train.loc[train['id'] == 1359,'budget'] = 10000000
train.loc[train['id'] == 1542,'budget'] = 1
train.loc[train['id'] == 1570,'budget'] = 15800000
train.loc[train['id'] == 1571,'budget'] = 4000000
train.loc[train['id'] == 1714,'budget'] = 46000000
train.loc[train['id'] == 1721,'budget'] = 17500000
train.loc[train['id'] == 1865,'revenue'] = 25000000
train.loc[train['id'] == 2268,'budget'] = 17500000
train.loc[train['id'] == 2491,'revenue'] = 6800000
train.loc[train['id'] == 2602,'budget'] = 31000000
train.loc[train['id'] == 2612,'budget'] = 15000000
train.loc[train['id'] == 2696,'budget'] = 10000000
train.loc[train['id'] == 2801,'budget'] = 10000000
test.loc[test['id'] == 3889,'budget'] = 15000000
test.loc[test['id'] == 6733,'budget'] = 5000000
test.loc[test['id'] == 3197,'budget'] = 8000000
test.loc[test['id'] == 6683,'budget'] = 50000000
test.loc[test['id'] == 5704,'budget'] = 4300000
test.loc[test['id'] == 6109,'budget'] = 281756
test.loc[test['id'] == 7242,'budget'] = 10000000
test.loc[test['id'] == 7021,'budget'] = 17540562
test.loc[test['id'] == 5591,'budget'] = 4000000
test.loc[test['id'] == 4282,'budget'] = 20000000
power_six = train.id[train.budget > 1000][train.revenue < 100]
for k in power_six :
train.loc[train['id'] == k,'revenue'] = train.loc[train['id'] == k,'revenue'] * 1000000
class PixelwiseNorm(nn.Module):
def __init__(self):
super(PixelwiseNorm, self ).__init__()
def forward(self, x, alpha=1e-8):

 y = x.pow(2.).mean(dim=1, keepdim=True ).add(alpha ).sqrt()
y = x / y
return y<categorify>
X = train.drop(['id', 'revenue'], axis=1)
 y = np.log1p(train['revenue'])
 X_test = test.drop(['id'], axis=1 )
class MinibatchStdDev(th.nn.Module):

 def __init__(self):

 super(MinibatchStdDev, self ).__init__()
def forward(self, x, alpha=1e-8):

 batch_size, _, height, width = x.shape
y = x - x.mean(dim=0, keepdim=True)
 y = th.sqrt(y.pow(2.).mean(dim=0, keepdim=False)+ alpha)
 y = y.mean().view(1, 1, 1, 1)
 y = y.repeat(batch_size,1, height, width)
 y = th.cat([x, y], 1)
 return y 
<define_search_model>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1 )
patchsize = 16
class Generator(nn.Module):
def __init__(self, nz, nfeats, nchannels):
super(Generator, self ).__init__()
ksize = 4
self.conv1 = spectral_norm(nn.ConvTranspose2d(nz, nfeats * 8, ksize, 1, 0, bias=False))
self.conv2 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 8, ksize, 2, 1, bias=False))
self.conv3 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 4, ksize, 2, 1, bias=False))
self.conv4 = spectral_norm(nn.ConvTranspose2d(nfeats * 4, nfeats * 2, ksize, 2, 1, bias=False))
self.conv5 = spectral_norm(nn.ConvTranspose2d(nfeats * 2, nfeats, ksize, 2, 1, bias=False))
self.conv6 = spectral_norm(nn.ConvTranspose2d(nfeats, nchannels, 3, 1, 1, bias=False))
self.pixnorm = PixelwiseNorm()
def forward(self, x):
x = F.leaky_relu(self.conv1(x))
x = F.leaky_relu(self.conv2(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv3(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv4(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv5(x))
x = self.pixnorm(x)
 x = torch.tanh(self.conv6(x))
return x
class Discriminator(nn.Module):
def __init__(self, nchannels, nfeats):
super(Discriminator, self ).__init__()
self.conv1 = nn.Conv2d(nchannels, nfeats, 4, 2, 1, bias=False)
 self.conv2 = spectral_norm(nn.Conv2d(nfeats, nfeats * 2, 4, 2, 1, bias=False))
self.bn2 = nn.BatchNorm2d(nfeats * 2)
 self.conv3 = spectral_norm(nn.Conv2d(nfeats * 2, nfeats * 4, 4, 2, 1, bias=False))
self.bn3 = nn.BatchNorm2d(nfeats * 4)
 self.conv4 = spectral_norm(nn.Conv2d(nfeats * 4, nfeats * 8, 4, 2, 1, bias=False))
self.bn4 = nn.MaxPool2d(2)
 self.batch_discriminator = MinibatchStdDev()
self.pixnorm = PixelwiseNorm()
self.conv5 = spectral_norm(nn.Conv2d(nfeats * 8 +1, patchsize, 2, 1, 0, bias=False))
def forward(self, x):
x = F.leaky_relu(self.conv1(x), 0.2)
 x = F.leaky_relu(self.bn2(self.conv2(x)) , 0.2)
 x = F.leaky_relu(self.bn3(self.conv3(x)) , 0.2)
 x = F.leaky_relu(self.bn4(self.conv4(x)) , 0.2)
 x = self.batch_discriminator(x)
 x = torch.sigmoid(self.conv5(x))
return x.view(-1, 1)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 lr = 0.0001
lr_d = 0.0003
beta1 = 0.5
epochs = 1600
nz = 196
netG = Generator(nz, 64, 3 ).to(device)
 netD = Discriminator(3, 64 ).to(device)
 criterion = nn.BCELoss()
optimizerD = optim.Adam(netD.parameters() , lr=lr_d, betas=(beta1, 0.999))
optimizerG = optim.Adam(netG.parameters() , lr=lr, betas=(beta1, 0.999))
lr_schedulerG = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerG,
T_0=epochs//200, eta_min=0.00005)
 lr_schedulerD = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerD,
T_0=epochs//200, eta_min=0.00005)
 fixed_noise = torch.randn(25, nz, 1, 1, device=device)
 batch_size = train_loader.batch_size
step = 0
for epoch in range(epochs):
for ii,(real_images)in tqdm(enumerate(train_loader), total=len(train_loader)) :
end = time()
if(end - start)> 25000 :
break
if step % 20 == 0:
real_label = 0.0
fake_label = 0.7
else:
real_label = 0.7
fake_label = 0.0
netD.zero_grad()
real_images = real_images.to(device)
 batch_size = real_images.size(0)
 labels = torch.full(( batch_size, patchsize), real_label, device=device ).view(-1, 1)+ np.random.uniform(-0.1, 0.1)
 output = netD(real_images)
 errD_real = criterion(output, labels)
 errD_real.backward()
D_x = output.mean().item()
noise = torch.randn(batch_size, nz, 1, 1, device=device)
 fake = netG(noise)
 labels.fill_(fake_label)+ np.random.uniform(0, 0.2)
 output = netD(fake.detach())
 errD_fake = criterion(output, labels)
 errD_fake.backward()
D_G_z1 = output.mean().item()
errD = errD_real + errD_fake
optimizerD.step()
netG.zero_grad()
labels.fill_(real_label)
 output = netD(fake)
 errG = criterion(output, labels)
 errG.backward()
D_G_z2 = output.mean().item()
optimizerG.step()
if step % 500 == 0:
print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)) : %.4f / %.4f'
%(epoch + 1, epochs, ii, len(train_loader),
errD.item() , errG.item() , D_x, D_G_z1, D_G_z2))
valid_image = netG(fixed_noise)
 step += 1
lr_schedulerG.step(epoch)
 lr_schedulerD.step(epoch)
 if epoch % 5 == 0:
show_generated_img()
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values
if not os.path.exists('.. /output_images'):
os.mkdir('.. /output_images')
 im_batch_size = 100
n_images=10000
for i_batch in range(0, n_images, im_batch_size):
z = truncated_normal(( im_batch_size, nz, 1, 1), threshold=1)
 gen_z = torch.from_numpy(z ).float().to(device)
 gen_images = netG(gen_z)
 images = gen_images.to("cpu" ).clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
 for i_image in range(gen_images.size(0)) :
save_image(( gen_images[i_image, :, :, :] +1.0)/2.0, os.path.join('.. /output_images', f'image_{i_batch+i_image:05d}.png'))
shutil.make_archive('images', 'zip', '.. /output_images' )<import_modules>
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
model1 = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model1.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200 )
from functools import partial
from multiprocessing import Pool
import os
from pathlib import Path
import random
import shutil
import time
import xml.etree.ElementTree as ET
from matplotlib import pyplot as plt
import numpy as np
from PIL import Image
from scipy.stats import truncnorm
import torch
from torch import nn, optim
from torch.optim.optimizer import Optimizer
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
from torchvision import datasets, transforms
from torchvision.datasets.folder import default_loader
from torchvision.utils import save_image
from torch.nn.utils import spectral_norm
from tqdm import tqdm
from torch.autograd import Variable
import sys<define_variables>
eli5.show_weights(model1, feature_filter=lambda x: x != '<BIAS>' )
start_time = time.time()
batch_size = 32
epochs = 210
seed = 1029
TRAIN_DIR = Path('.. /input/all-dogs/')
 ANNOTATION_DIR = Path('.. /input/annotation/Annotation/')
 DOG_DIR = Path('.. /dogs/dogs/')
 OUT_DIR = Path('.. /output_images/')
 DOG_DIR.mkdir(parents=True, exist_ok=True)
 OUT_DIR.mkdir(exist_ok=True)
 device = torch.device('cuda')
 lr = 0.0005
beta1 = 0.5
nz = 256
real_label = 0.95
fake_label = 0
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<define_variables>
n_fold = 10
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42 )
mylist = os.listdir(ANNOTATION_DIR )<define_variables>
def train_model(X, X_test, y, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, model=None):
oof = np.zeros(X.shape[0])
 prediction = np.zeros(X_test.shape[0])
 scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 if model_type == 'sklearn':
X_train, X_valid = X[train_index], X[valid_index]
else:
X_train, X_valid = X.values[train_index], X.values[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test.values), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_squared_error(y_valid, y_pred_valid)
 y_pred = model.predict(X_test)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric='RMSE', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_squared_error(y_valid, y_pred_valid)** 0.5)
 prediction += y_pred
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance
return oof, prediction
else:
return oof, prediction
label_map = {}
for listname in mylist:
label_map[listname.split('-')[0]]=listname.split('-')[1]<define_variables>
params = {'num_leaves': 30,
'min_data_in_leaf': 10,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
oof_lgb, prediction_lgb, _ = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=True )
label_number={}
for index, label in enumerate(label_map):
label_number[label]=index<normalization>
for col in train_texts.columns:
vectorizer = TfidfVectorizer(
 sublinear_tf=True,
analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
min_df=10 
)
 vectorizer.fit(list(train_texts[col].fillna('')) + list(test_texts[col].fillna('')))
 train_col_text = vectorizer.transform(train_texts[col].fillna(''))
test_col_text = vectorizer.transform(test_texts[col].fillna(''))
model = linear_model.RidgeCV(alphas=(0.01, 0.1, 1.0, 10.0, 100.0), scoring='neg_mean_squared_error', cv=folds)
 oof_text, prediction_text = train_model(train_col_text, test_col_text, y, params=None, model_type='sklearn', model=model)
 X[col + '_oof'] = oof_text
X_test[col + '_oof'] = prediction_text
class PixelwiseNorm(nn.Module):
def __init__(self):
super(PixelwiseNorm, self ).__init__()
def forward(self, x, alpha=1e-8):

 y = x.pow(2.).mean(dim=1, keepdim=True ).add(alpha ).sqrt()
y = x / y
return y<train_model>
def new_features(df):
df['budget_to_popularity'] = df['budget'] / df['popularity']
df['budget_to_runtime'] = df['budget'] / df['runtime']
df['_budget_year_ratio'] = df['budget'] /(df['release_date_year'] * df['release_date_year'])
 df['_releaseYear_popularity_ratio'] = df['release_date_year'] / df['popularity']
df['_releaseYear_popularity_ratio2'] = df['popularity'] / df['release_date_year']
df['runtime_to_mean_year'] = df['runtime'] / df.groupby("release_date_year")["runtime"].transform('mean')
 df['popularity_to_mean_year'] = df['popularity'] / df.groupby("release_date_year")["popularity"].transform('mean')
 df['budget_to_mean_year'] = df['budget'] / df.groupby("release_date_year")["budget"].transform('mean')
 return df
class ConditionalBatchNorm2d(nn.BatchNorm2d):

 def __init__(self, num_features, eps=1e-05, momentum=0.1,
affine=False, track_running_stats=True):
super(ConditionalBatchNorm2d, self ).__init__(
 num_features, eps, momentum, affine, track_running_stats 
)
 def forward(self, input, weight, bias, **kwargs):
self._check_input_dim(input)
 exponential_average_factor = 0.0
if self.training and self.track_running_stats:
self.num_batches_tracked += 1
if self.momentum is None:
exponential_average_factor = 1.0 / self.num_batches_tracked.item()
else:
exponential_average_factor = self.momentum
output = F.batch_norm(input, self.running_mean, self.running_var,
self.weight, self.bias,
self.training or not self.track_running_stats,
exponential_average_factor, self.eps)
 if weight.dim() == 1:
weight = weight.unsqueeze(0)
 if bias.dim() == 1:
bias = bias.unsqueeze(0)
 size = output.size()
weight = weight.unsqueeze(-1 ).unsqueeze(-1 ).expand(size)
 bias = bias.unsqueeze(-1 ).unsqueeze(-1 ).expand(size)
 return weight * output + bias
class CategoricalConditionalBatchNorm2d(ConditionalBatchNorm2d):
def __init__(self, num_classes, num_features, eps=1e-5, momentum=0.1,
affine=False, track_running_stats=True):
super(CategoricalConditionalBatchNorm2d, self ).__init__(
 num_features, eps, momentum, affine, track_running_stats 
)
 self.weights = nn.Embedding(num_classes, num_features)
 self.biases = nn.Embedding(num_classes, num_features)
 self._initialize()
def _initialize(self):
torch.nn.init.ones_(self.weights.weight.data)
 torch.nn.init.zeros_(self.biases.weight.data)
 def forward(self, input, c, **kwargs):
weight = self.weights(c)
 bias = self.biases(c)
 return super(CategoricalConditionalBatchNorm2d, self ).forward(input, weight, bias )<choose_model_class>
X = new_features(X)
 X_test = new_features(X_test )
class ResBlockGenerator(nn.Module):
def __init__(self, in_channels, out_channels, stride=1):
super(ResBlockGenerator, self ).__init__()
self.conv1 = nn.Conv2d(in_channels, out_channels, 3, 1, padding=1)
 self.conv2 = nn.Conv2d(out_channels, out_channels, 3, 1, padding=1)
 self.model = nn.Sequential(
 nn.BatchNorm2d(in_channels),
nn.ReLU() ,
nn.Upsample(scale_factor=2),
self.conv1,
nn.BatchNorm2d(out_channels),
nn.ReLU() ,
self.conv2 
)
 self.bypass = nn.Sequential()
if stride != 1:
if in_channels != out_channels:
self.bypass = nn.Sequential(
 nn.Upsample(scale_factor=2),
nn.Conv2d(in_channels,out_channels, 1, 1, padding=0)
)
 else:
self.bypass = nn.Upsample(scale_factor=2)
 def forward(self, x):
return self.model(x)+ self.bypass(x)
 class ResBlockDiscriminator(nn.Module):
def __init__(self, in_channels, out_channels, stride=1):
super(ResBlockDiscriminator, self ).__init__()
self.conv1 = nn.Conv2d(in_channels, out_channels, 3, 1, padding=1)
 self.conv2 = nn.Conv2d(out_channels, out_channels, 3, 1, padding=1)
 nn.init.xavier_uniform(self.conv1.weight.data, 1.)
nn.init.xavier_uniform(self.conv2.weight.data, 1.)
if stride == 1:
self.model = nn.Sequential(
 nn.ReLU() ,
spectral_norm(self.conv1),
nn.ReLU() ,
spectral_norm(self.conv2)
)
 else:
self.model = nn.Sequential(
 nn.ReLU() ,
spectral_norm(self.conv1),
nn.ReLU() ,
spectral_norm(self.conv2),
nn.AvgPool2d(2, stride=stride, padding=0)
)
 self.bypass = nn.Sequential()
if stride != 1:
self.bypass_conv = nn.Conv2d(in_channels,out_channels, 1, 1, padding=0)
 nn.init.xavier_uniform(self.bypass_conv.weight.data, np.sqrt(2))
self.bypass = nn.Sequential(
 spectral_norm(self.bypass_conv),
nn.AvgPool2d(2, stride=stride, padding=0)
)
 def forward(self, x):
return self.model(x)+ self.bypass(x )<normalization>
oof_lgb, prediction_lgb, _ = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=True )
class Generator(nn.Module):
def __init__(self, nz=128, channels=3):
super(Generator, self ).__init__()
self.nz = nz
self.channels = channels
def convlayer(n_input, n_output, k_size=4, stride=2, padding=0):
block = [
spectral_norm(nn.ConvTranspose2d(n_input, n_output, kernel_size=k_size, stride=stride, padding=padding, bias=False)) ,
nn.ReLU(inplace=True),
]
return block
self.layer1 = nn.Sequential(
 *convlayer(self.nz, 64, 4, 1, 0), 
)
 self.layer2 = nn.Sequential(ResBlockGenerator(64, 64, 2))
self.CBN2 = CategoricalConditionalBatchNorm2d(120,64)
 self.layer3 = nn.Sequential(ResBlockGenerator(64, 64, 2))
self.CBN3 = CategoricalConditionalBatchNorm2d(120,64)
 self.layer4 = nn.Sequential(ResBlockGenerator(64, 64, 2))
self.CBN4 = CategoricalConditionalBatchNorm2d(120,64)
 self.layer5 = nn.Sequential(ResBlockGenerator(64, 64, 2))
self.CBN5 = CategoricalConditionalBatchNorm2d(120,64)
 self.outlay = spectral_norm(nn.Conv2d(64, 3, 3, 1, 1))
self.Tanh = nn.Tanh()
def forward(self, z, y):
z = z.view(-1, self.nz, 1, 1)
 z = self.layer1(z)
 z = self.layer2(z)
 z = self.CBN2(z,y)
 z = self.layer3(z)
 z = self.CBN3(z,y)
 z = self.layer4(z)
 z = self.CBN4(z,y)
 z = self.layer5(z)
 z = self.CBN5(z,y)
 z = self.outlay(z)
 img = self.Tanh(z)
 return img
class Discriminator(nn.Module):
def __init__(self, channels=3):
super(Discriminator, self ).__init__()
self.channels = channels
def convlayer(n_input, n_output, k_size=4, stride=2, padding=0, bn=False):
block =[]
if bn:
block.append(spectral_norm(nn.Conv2d(n_input, n_output, kernel_size=k_size, stride=stride, padding=padding, bias=False)))
 block.append(nn.BatchNorm2d(n_output))
else:
block.append(nn.Conv2d(n_input, n_output, kernel_size=k_size, stride=stride, padding=padding, bias=False))
block.append(nn.LeakyReLU(0.2, inplace=True))
return block
self.model = nn.Sequential(
 *convlayer(self.channels, 64, 4, 2, 1),
ResBlockDiscriminator(64,128,2),
ResBlockDiscriminator(128,256,2),
ResBlockDiscriminator(256,728,2),
ResBlockDiscriminator(728,728,1), 
)
 self.logic_out = nn.Conv2d(728, 1, 4, 1, 0, bias=False)
 self.aux_out = nn.Conv2d(728, 120, 4, 1, 0, bias=False)
 def forward(self, imgs, label):
out = self.model(imgs)
 logic_output = self.logic_out(out)
 aux_output = self.aux_out(out)
 return logic_output.view(-1, 1),aux_output.view(-1,120 )<categorify>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1)
 params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 6,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
model1 = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model1.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200)
 eli5.show_weights(model1, feature_filter=lambda x: x != '<BIAS>' )
class DogsDataset(Dataset):
def __init__(self, root, annotation_root, transform=None,
target_transform=None, loader=default_loader, n_process=4):
self.root = Path(root)
 self.annotation_root = Path(annotation_root)
 self.transform = transform
self.target_transform = target_transform
self.loader = loader
self.imgs= self.cut_out_dogs(n_process)
 def _get_annotation_path(self, img_path):
dog = Path(img_path ).stem
breed = dog.split('_')[0]
breed_dir = next(self.annotation_root.glob(f'{breed}-*'))
return breed_dir / dog
@staticmethod
def _get_dog_box(annotation_path):
tree = ET.parse(annotation_path)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 yield(xmin, ymin, xmax, ymax)
 def crop_dog(self, path):
imgs = []
annotation_path = self._get_annotation_path(path)
 dog = Path(path ).stem
label = label_number[dog.split('_')[0]]
for bndbox in self._get_dog_box(annotation_path):
img = self.loader(path)
 img_ = img.crop(bndbox)
 if np.sum(img_)!= 0:
img = img_
imgs.append([img,label])
 return imgs
def label_dog(self, path):
label = []
dog = Path(path ).stem
label.append(label_number[dog.split('_')[0]])
 return label
def cut_out_dogs(self, n_process):
with Pool(n_process)as p:
imgs = p.map(self.crop_dog, self.root.iterdir())
 return imgs
def __getitem__(self, index):
if self.transform is not None:
samples = self.transform(self.imgs[index][0][0])
 lables = self.imgs[index][0][1]
return samples,lables
def __len__(self):
return len(self.imgs )<normalization>
def top_cols_interaction(df):
df['budget_to_year'] = df['budget'] / df['release_date_year']
df['budget_to_mean_year_to_year'] = df['budget_to_mean_year'] / df['release_date_year']
df['popularity_to_mean_year_to_log_budget'] = df['popularity_to_mean_year'] / df['log_budget']
df['year_to_log_budget'] = df['release_date_year'] / df['log_budget']
df['budget_to_runtime_to_year'] = df['budget_to_runtime'] / df['release_date_year']
df['genders_1_cast_to_log_budget'] = df['genders_1_cast'] / df['log_budget']
df['all_genres_to_popularity_to_mean_year'] = df['all_genres'] / df['popularity_to_mean_year']
df['genders_2_crew_to_budget_to_mean_year'] = df['genders_2_crew'] / df['budget_to_mean_year']
df['overview_oof_to_genders_2_crew'] = df['overview_oof'] / df['genders_2_crew']
return df
class ParamScheduler(object):
def __init__(self, optimizer, scale_fn, step_size):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
self.scale_fn = scale_fn
self.step_size = step_size
self.last_batch_iteration = 0
def batch_step(self):
for param_group in self.optimizer.param_groups:
param_group['lr'] = self.scale_fn(self.last_batch_iteration / self.step_size)
 self.last_batch_iteration += 1
def combine_scale_functions(scale_fns, phases=None):
if phases is None:
phases = [1./ len(scale_fns)] * len(scale_fns)
 phases = [phase / sum(phases)for phase in phases]
phases = torch.tensor([0] + phases)
 phases = torch.cumsum(phases, 0)
 def _inner(x):
idx =(x >= phases ).nonzero().max()
actual_x =(x - phases[idx])/(phases[idx + 1] - phases[idx])
 return scale_fns[idx](actual_x)
 return _inner
def scale_cos(start, end, x):
return start +(1 + np.cos(np.pi *(1 - x)))*(end - start)/ 2<normalization>
X = top_cols_interaction(X)
 X_test = top_cols_interaction(X_test )
random_transforms = [transforms.RandomRotation(degrees=5)]
transform = transforms.Compose([transforms.Resize(64),
transforms.CenterCrop(64),
transforms.RandomHorizontalFlip(p=0.5),
transforms.RandomApply(random_transforms, p=0.3),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 train_data = DogsDataset(TRAIN_DIR / 'all-dogs/', ANNOTATION_DIR, transform=transform)
 train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size, num_workers=4)
<define_variables>
X = X.replace([np.inf, -np.inf], 0 ).fillna(0)
 X_test = X_test.replace([np.inf, -np.inf], 0 ).fillna(0 )
imgs,labels = next(iter(train_loader))<choose_model_class>
trainAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TrainAdditionalFeatures.csv')
 testAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TestAdditionalFeatures.csv')
 train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 X['imdb_id'] = train['imdb_id']
X_test['imdb_id'] = test['imdb_id']
del train, test
X = pd.merge(X, trainAdditionalFeatures, how='left', on=['imdb_id'])
 X_test = pd.merge(X_test, testAdditionalFeatures, how='left', on=['imdb_id'])
 X = X.drop(['imdb_id'], axis=1)
 X_test = X_test.drop(['imdb_id'], axis=1 )
net_g = Generator(nz ).to(device)
 net_d = Discriminator().to(device)
 criterion = nn.BCELoss()
auxiliary_loss = torch.nn.CrossEntropyLoss()
scale_fn = combine_scale_functions(
 [partial(scale_cos, 1e-5, 5e-4), partial(scale_cos, 5e-4, 1e-4)], [0.2, 0.8])
 optimizer_g = optim.Adam(net_g.parameters() , lr=lr, betas=(beta1, 0.999))
optimizer_d = optim.Adam(net_d.parameters() , lr=lr, betas=(beta1, 0.999))
scheduler_g = ParamScheduler(optimizer_g, scale_fn, epochs * len(train_loader))
scheduler_d = ParamScheduler(optimizer_d, scale_fn, epochs * len(train_loader))<categorify>
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 9,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
oof_lgb, prediction_lgb, _ = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=True )
for epoch in range(epochs):
for i,(real_images,label)in enumerate(train_loader):
net_d.zero_grad()
real_images = real_images.to(device)
 label = label.type(torch.LongTensor ).to(device)
 batch_size = real_images.size(0)
 labels = torch.full(( batch_size, 1), real_label, device=device)
 scheduler_d.batch_step()
output_real,aux_real = net_d(real_images,label)
 noise = torch.randn(batch_size, nz, 1, 1, device=device)
 fake = net_g(noise,label)
 output_fake,aux_fake = net_d(fake.detach() ,label)
 loss_aux_real = auxiliary_loss(aux_real,label)
 loss_aux_fake = auxiliary_loss(aux_fake,label)
 err_d =(torch.mean(( output_real - torch.mean(output_fake)- labels)** 2)+
torch.mean(( output_fake - torch.mean(output_real)+ labels)** 2)) / 2
err_d = err_d*0.7 + loss_aux_real*0.15 + loss_aux_fake*0.15
err_d.backward(retain_graph=True)
 optimizer_d.step()
net_g.zero_grad()
scheduler_g.batch_step()
output_fake, aux_fake = net_d(fake,label)
 loss_aux_fake_g = auxiliary_loss(aux_fake,label)
 err_g =(torch.mean(( output_real - torch.mean(output_fake)+ labels)** 2)+
torch.mean(( output_fake - torch.mean(output_real)- labels)** 2)) / 2
err_g = err_g*0.85 + loss_aux_fake_g*0.15
err_g.backward()
optimizer_g.step()
print(f'[{epoch + 1}/{epochs}] Loss_d: {err_d.item() :.4f} Loss_g: {err_g.item() :.4f}' )<compute_test_metric>
xgb_params = {'eta': 0.01,
'objective': 'reg:linear',
'max_depth': 7,
'subsample': 0.8,
'colsample_bytree': 0.8,
'eval_metric': 'rmse',
'seed': 11,
'silent': True}
oof_xgb, prediction_xgb = train_model(X, X_test, y, params=xgb_params, model_type='xgb', plot_feature_importance=False )
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values<categorify>
cat_params = {'learning_rate': 0.002,
'depth': 5,
'l2_leaf_reg': 10,
'colsample_bylevel': 0.8,
'bagging_temperature': 0.2,
'od_type': 'Iter',
'od_wait': 100,
'random_seed': 11,
'allow_writing_files': False}
oof_cat, prediction_cat = train_model(X, X_test, y, params=cat_params, model_type='cat' )
im_batch_size = 50
n_images = 10000
for i_batch in range(0, n_images, im_batch_size):
z = truncated_normal(( im_batch_size, nz, 1, 1), threshold=1)
 gen_z = torch.from_numpy(z ).float().to(device)
 gen_labels = Variable(torch.LongTensor(np.random.randint(0, 120, im_batch_size)) ).to(device)
 gen_images =(net_g(gen_z,gen_labels)+ 1)/ 2
images = gen_images.to('cpu' ).clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
 for i_image in range(gen_images.size(0)) :
save_image(gen_images[i_image, :, :, :], OUT_DIR / f'image_{i_batch + i_image:05d}.png' )<load_pretrained>
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
oof_lgb_1, prediction_lgb_1 = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=False )
shutil.make_archive('images', 'zip', OUT_DIR )<set_options>
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 7,
'learning_rate': 0.02,
"boosting": "gbdt",
"feature_fraction": 0.7,
"bagging_freq": 5,
"bagging_fraction": 0.7,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
oof_lgb_2, prediction_lgb_2 = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=False )
writer = tbx.SummaryWriter("." )<define_variables>
train_stack = np.vstack([oof_lgb, oof_xgb, oof_cat, oof_lgb_1, oof_lgb_2] ).transpose()
train_stack = pd.DataFrame(train_stack, columns=['lgb', 'xgb', 'cat', 'lgb_1', 'lgb_2'])
 test_stack = np.vstack([prediction_lgb, prediction_xgb, prediction_cat, prediction_lgb_1, prediction_lgb_2] ).transpose()
test_stack = pd.DataFrame(test_stack, columns=['lgb', 'xgb', 'cat', 'lgb_1', 'lgb_2'] )
start_time = time.time()
random.seed(0)
 np.random.seed(0)
 torch.manual_seed(0 )<load_pretrained>
model = linear_model.RidgeCV(alphas=(0.01, 0.1, 1.0, 10.0, 100.0), scoring='neg_mean_squared_error', cv=folds)
 oof_rcv_stack, prediction_rcv_stack = train_model(train_stack.values, test_stack.values, y, params=None, model_type='sklearn', model=model )
def doggo_loader(path):
img = torchvision.datasets.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(dirname for dirname in os.listdir('.. /input/annotation/Annotation/')if dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join('.. /input/annotation/Annotation', annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for idx, o in enumerate(objects):
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = xmax - xmin
h = ymax - ymin
l = max([w,h])
 if w > h:
ymin = ymin - int(( l-h)/2)
 else:
xmin = xmin - int(( l-w)/2)
 bbox =(xmin, ymin , xmin+l, ymin+l)
 c_img = img.crop(bbox ).copy()
c_img.thumbnail(( 100, 100), Image.ANTIALIAS)
 c_img.save('.. /class_images/train/image_{}_{}.png'.format(annotation_basename, idx))
return img
dataset = torchvision.datasets.ImageFolder(
 '.. /input/all-dogs/',
loader=doggo_loader,
transform=torchvision.transforms.Compose([
transforms.RandomResizedCrop(64, scale=(1.0, 1.0), ratio=(1., 1.)),
torchvision.transforms.ToTensor() ,
])
 )<load_pretrained>
sub = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 sub['revenue'] = np.expm1(prediction_lgb)
 sub.to_csv("lgb.csv", index=False)
 sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb)/ 2)
 sub.to_csv("blend.csv", index=False)
 sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb + prediction_cat)/ 3)
 sub.to_csv("blend1.csv", index=False)
 sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb + prediction_cat + prediction_lgb_1)/ 4)
 sub.to_csv("blend2.csv", index=False)
 sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb + prediction_cat + prediction_lgb_1 + prediction_lgb_2)/ 5)
 sub.to_csv("blend3.csv", index=False)
 sub['revenue'] = prediction_lgb_stack
sub.to_csv("stack_lgb.csv", index=False)
 sub['revenue'] = prediction_rcv_stack
sub.to_csv("stack_rcv.csv", index=False )