Kaggle competition: COVID19 Global Forecasting (Week 2)
def get_features(path):
try:
st = []
img = Image.open(path)
 im_stats_ = ImageStat.Stat(img)
 st += im_stats_.sum
st += im_stats_.mean
st += im_stats_.rms
st += im_stats_.var
st += im_stats_.stddev
except:
print(path)
 return [path, st]
def normalize_img(paths):
imf_d = {}
p = Pool(cpu_count())
 ret = p.map(get_features, paths)
 for i in range(len(ret)) :
imf_d[ret[i][0]] = ret[i][1]
ret = []
fdata = [imf_d[f] for f in paths]
return pd.DataFrame(fdata)
 dog_bytes = pd.DataFrame(glob.glob('.. /input/all-dogs/all-dogs/**'), columns=['Path'])
 dog_bytes = pd.concat(( dog_bytes, normalize_img(dog_bytes.Path.values)) , axis=1)
 dog_bytes.head()<predict_on_test>
import matplotlib.pyplot as plt
from typing import Tuple
dog_bytes['Group'] = cluster.KMeans(n_clusters=100, random_state=3, n_jobs=-1 ).fit_predict(dog_bytes[list(range(15)) ])
 dog_bytes['Group'].value_counts() [:5]<define_variables>
fname_train = '.. /input/covid19-global-forecasting-week-2/train.csv'
fname_test = '.. /input/covid19-global-forecasting-week-2/test.csv'
fname_sub = '.. /input/covid19-global-forecasting-week-2/submission.csv'
%matplotlib inline
fig = plt.figure(figsize=(8, 80))
samples = []
for i in range(100):
g = dog_bytes[dog_bytes['Group'] == i]
if len(g)>= 5:
samples += list(g['Path'].values[:5])
 for i in range(len(samples)) [:50]:
ax = fig.add_subplot(len(samples)/5, 5, i+1, xticks=[], yticks=[])
 img = Image.open(samples[i])
 img = img.resize(( 100,int(img.size[1]/(img.size[0]/100))), Image.ANTIALIAS)
 img = img.crop(( 0, 0, 64, 64))
plt.imshow(img )<statistical_test>
df_train = pd.read_csv(fname_train,parse_dates=True)
 df_test = pd.read_csv(fname_test, parse_dates=True)
 df_sub = pd.read_csv(fname_sub )
def sim_img(path):
img = Image.open(path ).convert('RGB')
 img = img.resize(( 100,int(img.size[1]/(img.size[0]/100))), Image.ANTIALIAS)
 img = img.crop(( 0, 0, 64, 64))
return img
samples = []
for i in range(100):
g = dog_bytes[dog_bytes['Group'] == i]
if len(g)>= 23:
s = g['Path'].values[:23]
s = list([p for p,_ in itertools.groupby(sorted([sorted(p)for p in list(itertools.permutations(s, 2)) ])) ])
 samples += s
print(len(samples))<init_hyperparams>
START_TRAIN_DATE = df_train['Date'].min()
END_TRAIN_DATE = df_train['Date'].max()
START_PREDICTION_DATE = df_test['Date'].min()
END_PREDICTION_DATE = df_test['Date'].max()
batch_size = 16
nz = 100
lr = 0.0003
lr_d = 0.0001
beta1 = 0.5
epochs = 198
t_0_num = 8
real_label = 0.7
fake_label = 0.0<init_hyperparams>
fname_pop = '.. /input/population-by-country-2020/population_by_country_2020.csv'
df_pop = pd.read_csv(fname_pop )
class AdaBound(Optimizer):

 def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), final_lr=0.1, gamma=1e-3,
eps=1e-8, weight_decay=0, amsbound=False):
if not 0.0 <= lr:
raise ValueError("Invalid learning rate: {}".format(lr))
if not 0.0 <= eps:
raise ValueError("Invalid epsilon value: {}".format(eps))
if not 0.0 <= betas[0] < 1.0:
raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
if not 0.0 <= betas[1] < 1.0:
raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
if not 0.0 <= final_lr:
raise ValueError("Invalid final learning rate: {}".format(final_lr))
if not 0.0 <= gamma < 1.0:
raise ValueError("Invalid gamma parameter: {}".format(gamma))
defaults = dict(lr=lr, betas=betas, final_lr=final_lr, gamma=gamma, eps=eps,
weight_decay=weight_decay, amsbound=amsbound)
 super(AdaBound, self ).__init__(params, defaults)
 self.base_lrs = list(map(lambda group: group['lr'], self.param_groups))
def __setstate__(self, state):
super(AdaBound, self ).__setstate__(state)
 for group in self.param_groups:
group.setdefault('amsbound', False)
 def step(self, closure=None):

 loss = None
if closure is not None:
loss = closure()
for group, base_lr in zip(self.param_groups, self.base_lrs):
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data
if grad.is_sparse:
raise RuntimeError(
 'Adam does not support sparse gradients, please consider SparseAdam instead')
 amsbound = group['amsbound']
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p.data)
 state['exp_avg_sq'] = torch.zeros_like(p.data)
 if amsbound:
state['max_exp_avg_sq'] = torch.zeros_like(p.data)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
if amsbound:
max_exp_avg_sq = state['max_exp_avg_sq']
beta1, beta2 = group['betas']
state['step'] += 1
if group['weight_decay'] != 0:
grad = grad.add(group['weight_decay'], p.data)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 if amsbound:
torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)
 denom = max_exp_avg_sq.sqrt().add_(group['eps'])
 else:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 bias_correction1 = 1 - beta1 ** state['step']
bias_correction2 = 1 - beta2 ** state['step']
step_size = group['lr'] * math.sqrt(bias_correction2)/ bias_correction1
final_lr = group['final_lr'] * group['lr'] / base_lr
lower_bound = final_lr *(1 - 1 /(group['gamma'] * state['step'] + 1))
upper_bound = final_lr *(1 + 1 /(group['gamma'] * state['step']))
step_size = torch.full_like(denom, step_size)
 step_size.div_(denom ).clamp_(lower_bound, upper_bound ).mul_(exp_avg)
 p.data.add_(-step_size)
 return loss
class AdaBoundW(Optimizer):

 def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), final_lr=0.1, gamma=1e-3,
eps=1e-8, weight_decay=0, amsbound=False):
if not 0.0 <= lr:
raise ValueError("Invalid learning rate: {}".format(lr))
if not 0.0 <= eps:
raise ValueError("Invalid epsilon value: {}".format(eps))
if not 0.0 <= betas[0] < 1.0:
raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
if not 0.0 <= betas[1] < 1.0:
raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
if not 0.0 <= final_lr:
raise ValueError("Invalid final learning rate: {}".format(final_lr))
if not 0.0 <= gamma < 1.0:
raise ValueError("Invalid gamma parameter: {}".format(gamma))
defaults = dict(lr=lr, betas=betas, final_lr=final_lr, gamma=gamma, eps=eps,
weight_decay=weight_decay, amsbound=amsbound)
 super(AdaBoundW, self ).__init__(params, defaults)
 self.base_lrs = list(map(lambda group: group['lr'], self.param_groups))
def __setstate__(self, state):
super(AdaBoundW, self ).__setstate__(state)
 for group in self.param_groups:
group.setdefault('amsbound', False)
 def step(self, closure=None):

 loss = None
if closure is not None:
loss = closure()
for group, base_lr in zip(self.param_groups, self.base_lrs):
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data
if grad.is_sparse:
raise RuntimeError(
 'Adam does not support sparse gradients, please consider SparseAdam instead')
 amsbound = group['amsbound']
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p.data)
 state['exp_avg_sq'] = torch.zeros_like(p.data)
 if amsbound:
state['max_exp_avg_sq'] = torch.zeros_like(p.data)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
if amsbound:
max_exp_avg_sq = state['max_exp_avg_sq']
beta1, beta2 = group['betas']
state['step'] += 1
exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 if amsbound:
torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)
 denom = max_exp_avg_sq.sqrt().add_(group['eps'])
 else:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 bias_correction1 = 1 - beta1 ** state['step']
bias_correction2 = 1 - beta2 ** state['step']
step_size = group['lr'] * math.sqrt(bias_correction2)/ bias_correction1
final_lr = group['final_lr'] * group['lr'] / base_lr
lower_bound = final_lr *(1 - 1 /(group['gamma'] * state['step'] + 1))
upper_bound = final_lr *(1 + 1 /(group['gamma'] * state['step']))
step_size = torch.full_like(denom, step_size)
 step_size.div_(denom ).clamp_(lower_bound, upper_bound ).mul_(exp_avg)
 if group['weight_decay'] != 0:
decayed_weights = torch.mul(p.data, group['weight_decay'])
 p.data.add_(-step_size)
 p.data.sub_(decayed_weights)
 else:
p.data.add_(-step_size)
 return loss 
<import_modules>
GLOBAL_FILL_STRING = 'not_provided'
import matplotlib.pyplot as plt
import numpy as np
import torch
import pandas as pd
from torch import nn, optim
import torch.nn.functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
import os
from torch import nn, optim
import torch.nn.functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
from torch.utils.data import Dataset, DataLoader
import os
from tqdm import tqdm_notebook as tqdm
from time import time
from PIL import Image
import torch.nn.parallel
import torch.optim as optim
import torch.utils.data
import torchvision
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable
from torch import nn, optim
import torch.nn.functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
import matplotlib.image as mpimg
import torchvision
import torchvision.datasets as dset
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable
import xml.etree.ElementTree as ET
import random
from torch.nn.utils import spectral_norm
from scipy.stats import truncnorm
import torch as th
import cv2 
<define_variables>
prediction_scores = {}
start = time()<set_options>
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<choose_model_class>
golden_data = {'DateTime':pd.to_datetime(['2020-06-01','2020-06-02','2020-06-03']),
'ConfirmedCases':[2,4,8],
'Fatalities':[0,0,1],
'Province_State':['PS_TBD','PS_TBD','PS_TBD'],
'Country_Region':['CR_TBD','CR_TBD','CR_TBD'],
'Id':[201,202,203],
'SimpleName':['TBD','TBD','TBD']}
predicted_data = {'DateTime':pd.to_datetime(['2020-06-01','2020-06-02','2020-06-03','2020-06-04','2020-06-05']),
'ConfirmedCases':[2,4,6,8,10],
'Fatalities':[0,0,0,1,2],
'SimpleName':['TBD','TBD','TBD','TBD','TBD']}
df_golden = pd.DataFrame.from_dict(golden_data)
 df_predicted = pd.DataFrame.from_dict(predicted_data)
 unit_c, unit_f = evaluate_predictions(df_golden,df_predicted,1,2,'unit test',False)
 assert(( unit_c == 0.1777)&(unit_f == 0.4901)) ,'Failed unit test for evaluate_predictions'
cascade_file_path = '/opt/conda/lib/python3.6/site-packages/cv2/data/haarcascade_{}.xml'
ordered_classifiers = [
cv2.CascadeClassifier(cascade_file_path.format('upperbody')) ,
cv2.CascadeClassifier(cascade_file_path.format('lowerbody')) ,
cv2.CascadeClassifier(cascade_file_path.format('frontalface_default')) ,
cv2.CascadeClassifier(cascade_file_path.format('frontalface_alt')) ,
cv2.CascadeClassifier(cascade_file_path.format('frontalface_alt2')) ,
]
def check_human(img):
img = np.asarray(img)
 for clf in ordered_classifiers:
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
 pos = clf.detectMultiScale(gray)
 if len(pos)!= 0:
return False
return True<load_pretrained>
df_train[['Province_State','Country_Region']] = df_train[['Province_State','Country_Region']].fillna(GLOBAL_FILL_STRING)
 df_test[['Province_State','Country_Region']] = df_test[['Province_State','Country_Region']].fillna(GLOBAL_FILL_STRING )
class DataGenerator(Dataset):
def __init__(self, directory, transform=None, n_samples=np.inf):
self.directory = directory
self.transform = transform
self.n_samples = n_samples
self.samples = self._load_subfolders_images(directory)
 if len(self.samples)== 0:
raise RuntimeError("Found 0 files in subfolders of: {}".format(directory))
def _load_subfolders_images(self, root):
IMG_EXTENSIONS =(
 '.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
 def is_valid_file(x):
return torchvision.datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 required_transforms = torchvision.transforms.Compose([
torchvision.transforms.Resize(64),
torchvision.transforms.CenterCrop(64),
])
 imgs = []
paths = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames)[:min(self.n_samples, 999999999999999)]:
path = os.path.join(root, fname)
 paths.append(path)
 for path in paths:
if is_valid_file(path):
img = dset.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(
 dirname for dirname in os.listdir('.. /input/annotation/Annotation/')if
dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join('.. /input/annotation/Annotation/',
annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
bbox =(xmin, ymin, xmin+w, ymin+w)
 object_img = required_transforms(img.crop(bbox))
if check_human(object_img):
imgs.append(object_img)
 return imgs
def __getitem__(self, index):
sample = self.samples[index]
if self.transform is not None:
sample = self.transform(sample)
 return np.asarray(sample)
 def __len__(self):
return len(self.samples)
<categorify>
df_train['SimpleName'] = df_train['Province_State'] + '__' + df_train['Country_Region']
df_test['SimpleName'] = df_test['Province_State'] + '__' + df_test['Country_Region']
%%time
database = '.. /input/all-dogs/all-dogs/'
transform = transforms.Compose([transforms.RandomHorizontalFlip(p=0.3),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 train_data = DataGenerator(database, transform=transform,n_samples=25000)
 train_loader = torch.utils.data.DataLoader(train_data, shuffle=True,batch_size=batch_size, num_workers = 4)
<normalization>
df_train['DateTime'] = pd.to_datetime(df_train['Date'])
 df_test['DateTime'] = pd.to_datetime(df_test['Date'] )
class PixelwiseNorm(nn.Module):
def __init__(self):
super(PixelwiseNorm, self ).__init__()
def forward(self, x, alpha=1e-8):

 y = x.pow(2.).mean(dim=1, keepdim=True ).add(alpha ).sqrt()
y = x / y
return y<categorify>
df_train['SimpleName'].nunique()
class MinibatchStdDev(th.nn.Module):

 def __init__(self):

 super(MinibatchStdDev, self ).__init__()
def forward(self, x, alpha=1e-8):

 batch_size, _, height, width = x.shape
y = x - x.mean(dim=0, keepdim=True)
 y = th.sqrt(y.pow(2.).mean(dim=0, keepdim=False)+ alpha)
 y = y.mean().view(1, 1, 1, 1)
 y = y.repeat(batch_size,1, height, width)
 y = th.cat([x, y], 1)
 return y 
<define_search_model>
sname_set = list(set(df_train['SimpleName']))
class Generator(nn.Module):
def __init__(self, nz, nfeats, nchannels):
super(Generator, self ).__init__()
self.conv1 = spectral_norm(nn.ConvTranspose2d(nz, nfeats * 8, 4, 1, 0, bias=False))
self.conv2 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 8, 4, 2, 1, bias=False))
self.conv3 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 4, 4, 2, 1, bias=False))
self.conv4 = spectral_norm(nn.ConvTranspose2d(nfeats * 4, nfeats * 2, 4, 2, 1, bias=False))
self.conv5 = spectral_norm(nn.ConvTranspose2d(nfeats * 2, nfeats, 4, 2, 1, bias=False))
self.conv6 = spectral_norm(nn.ConvTranspose2d(nfeats, nchannels, 3, 1, 1, bias=False))
self.pixnorm = PixelwiseNorm()
def forward(self, x):
x = F.leaky_relu(self.conv1(x))
x = F.leaky_relu(self.conv2(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv3(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv4(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv5(x))
x = self.pixnorm(x)
 x = torch.tanh(self.conv6(x))
return x
class Discriminator(nn.Module):
def __init__(self, nchannels, nfeats):
super(Discriminator, self ).__init__()
self.conv1 = nn.Conv2d(nchannels, nfeats, 4, 2, 1, bias=False)
 self.conv2 = spectral_norm(nn.Conv2d(nfeats, nfeats * 2, 4, 2, 1, bias=False))
self.bn2 = nn.BatchNorm2d(nfeats * 2)
 self.conv3 = spectral_norm(nn.Conv2d(nfeats * 2, nfeats * 4, 4, 2, 1, bias=False))
self.bn3 = nn.BatchNorm2d(nfeats * 4)
 self.conv4 = spectral_norm(nn.Conv2d(nfeats * 4, nfeats * 8, 4, 2, 1, bias=False))
self.bn4 = nn.MaxPool2d(2)
 self.batch_discriminator = MinibatchStdDev()
self.pixnorm = PixelwiseNorm()
self.conv5 = spectral_norm(nn.Conv2d(nfeats * 8 +1, 1, 2, 1, 0, bias=False))
def forward(self, x):
x = F.leaky_relu(self.conv1(x), 0.2)
 x = F.leaky_relu(self.bn2(self.conv2(x)) , 0.2)
 x = F.leaky_relu(self.bn3(self.conv3(x)) , 0.2)
 x = F.leaky_relu(self.bn4(self.conv4(x)) , 0.2)
 x = self.batch_discriminator(x)
 x = torch.sigmoid(self.conv5(x))
return x.view(-1, 1)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 netG = Generator(nz, 32, 3 ).to(device)
 netD = Discriminator(3, 48 ).to(device)
 criterion = nn.BCELoss()
optimizerD = AdaBound(netD.parameters() , lr=lr, betas=(beta1, 0.999))
optimizerG = AdaBound(netG.parameters() , lr=lr_d, betas=(beta1, 0.999))
lr_schedulerG = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerG,
T_0=t_0_num, eta_min=0.00005)
 lr_schedulerD = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerD,
T_0=t_0_num, eta_min=0.00005)
 fixed_noise = torch.randn(25, nz, 1, 1, device=device)
 batch_size = train_loader.batch_size
last_epoch = 0
step = 0
for epoch in range(epochs):
for ii,(real_images)in tqdm(enumerate(train_loader), total=len(train_loader)) :
end = time()
if(end -start)> 30000 :
break
netD.zero_grad()
real_images = real_images.to(device)
 batch_size = real_images.size(0)
 labels = torch.full(( batch_size, 1), real_label, device=device)+ np.random.uniform(-0.1, 0.1)
 output = netD(real_images)
 errD_real = criterion(output, labels)
 errD_real.backward()
D_x = output.mean().item()
noise = torch.randn(batch_size, nz, 1, 1, device=device)
 fake = netG(noise)
 labels.fill_(fake_label)+ np.random.uniform(0, 0.2)
 output = netD(fake.detach())
 errD_fake = criterion(output, labels)
 errD_fake.backward()
D_G_z1 = output.mean().item()
errD = errD_real + errD_fake
optimizerD.step()
netG.zero_grad()
labels.fill_(real_label)
 output = netD(fake)
 errG = criterion(output, labels)
 errG.backward()
D_G_z2 = output.mean().item()
optimizerG.step()
if step % 500 == 0:
print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)) : %.4f / %.4f'
%(epoch + 1, epochs, ii, len(train_loader),
errD.item() , errG.item() , D_x, D_G_z1, D_G_z2))
step += 1
lr_schedulerG.step(epoch)
 lr_schedulerD.step(epoch)
 last_epoch = epoch 
<categorify>
assert(0 == check_first_ConfirmedCase_exists(df_train,sname_set)) ,"Data contains locations with NO ConfirmedCases!"
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values
if not os.path.exists('.. /output_images'):
os.mkdir('.. /output_images')
 im_batch_size = 100
n_images=10000
for i_batch in range(0, n_images, im_batch_size):
z = truncated_normal(( im_batch_size, nz, 1, 1), threshold=1)
 gen_z = torch.from_numpy(z ).float().to(device)
 gen_images = netG(gen_z)
 images = gen_images.to("cpu" ).clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
 for i_image in range(gen_images.size(0)) :
save_image(( gen_images[i_image, :, :, :] +1.0)/2.0, os.path.join('.. /output_images', f'image_{i_batch+i_image:05d}.png'))
shutil.make_archive('images', 'zip', '.. /output_images' )<import_modules>
parameter_data = create_parameter_database()
db_parameters = pd.DataFrame(parameter_data, index=sname_set,\
columns=['beta','gamma','ifx','rfx','population'] )
import gzip, pickle
import os
import numpy as np
import pandas as pd
import random
import shutil
import numpy as np
import os
import torch
from torch import nn, optim
import torch.nn.functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
from torch.utils.data import Dataset, DataLoader
from scipy import linalg
import pathlib
import urllib
import warnings
from PIL import Image
from tqdm import tqdm_notebook as tqdm<define_variables>
ctry_col = r'Country(or dependency)'
pop_col = r'Population(2020)'
ComputeLB = True
DogsOnly = False
ROOT = '.. /input/generative-dog-images/'
ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 namesIn.append(breed)
 idxIn += 1
idx = np.arange(idxIn)
 np.random.shuffle(idx)
 imagesIn = imagesIn[idx,:,:,:]
namesIn = np.array(namesIn)[idx]
else:
IMAGES = np.sort(IMAGES)
 np.random.seed(810)
 x = np.random.choice(np.arange(20579),10000)
 np.random.seed(None)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]; h = img.size[1];
if(k%2==0)|(k%3==0):
w2 = 100; h2 = int(h/(w/100))
a = 18; b = 0
else:
a=0; b=0
if w<h:
w2 = 64; h2 = int(( 64/w)*h)
 b =(h2-64)//2
else:
h2 = 64; w2 = int(( 64/h)*w)
 a =(w2-64)//2
img = img.resize(( w2,h2), Image.ANTIALIAS)
 img = img.crop(( 0+a, 0+b, 64+a, 64+b))
imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 idxIn += 1
x = np.random.randint(0,idxIn,25)
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = Image.fromarray(imagesIn[x[k*5+j],:,:,:].astype('uint8'))
plt.axis('off')
 if not DogsOnly: plt.title(namesIn[x[k*5+j]],fontsize=11)
 else: plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)
 plt.imshow(img)
 plt.show()<categorify>
df_pop[df_pop[ctry_col] == 'France'][pop_col].values[0]
class dog_dataset(Dataset):
def __init__(self, train_y, train_X, zeros, device):
self.train_y = torch.Tensor(train_y ).to(device)
 self.train_X = torch.Tensor(train_X ).to(device)
 self.zeros = torch.Tensor(zeros ).to(device)
 def __len__(self):
return len(train_y)
 def __getitem__(self, idx):
return self.train_y[idx], self.train_X[idx], self.zeros[idx]<prepare_x_and_y>
for index,row in db_parameters.iterrows() :
state, ctry = index.split('__')
 if(state == GLOBAL_FILL_STRING):
try:
pop = df_pop[df_pop[ctry_col] == ctry][pop_col].values[0]
except:
if(ctry == 'Czechia'):
pop = df_pop[df_pop[ctry_col] == r'Czech Republic(Czechia)'][pop_col].values[0]
elif(ctry == r'Congo(Kinshasa)'):
pop = df_pop[df_pop[ctry_col] == r'DR Congo'][pop_col].values[0]
elif(ctry == r'Congo(Brazzaville)'):
pop = df_pop[df_pop[ctry_col] == r'Congo'][pop_col].values[0]
elif(ctry == r'Saint Vincent and the Grenadines'):
pop = df_pop[df_pop[ctry_col] == r'St.Vincent & Grenadines'][pop_col].values[0]
elif(ctry == r'Korea, South'):
pop = df_pop[df_pop[ctry_col] == r'South Korea'][pop_col].values[0]
elif(ctry == r"Cote d'Ivoire"):
pop = df_pop[df_pop[ctry_col] == r"Côte d'Ivoire"][pop_col].values[0]
elif(ctry == r'Saint Kitts and Nevis'):
pop = df_pop[df_pop[ctry_col] == r'Saint Kitts & Nevis'][pop_col].values[0]
elif(ctry == r'Taiwan*'):
pop = df_pop[df_pop[ctry_col] == r'Taiwan'][pop_col].values[0]
elif(ctry == r'Diamond Princess'):
pop = 3711
else:
pop=500000
print("No population data available for {}, {}".format(state,ctry))
db_parameters.loc[index,'population'] = pop
else:
print("State missing population data: {}, {}".format(state,ctry))
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 bs = 256
train_y =(imagesIn[:10000,:,:,:]/255.).reshape(( -1,12288))
train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1
zeros = np.zeros(( 10000,12288))
data_set = dog_dataset(train_y, train_X, zeros, device)
 data_loader = DataLoader(data_set, bs )<normalization>
col_len = df_test.shape[0]
zeros_column = np.zeros(( col_len,))
df_test['ConfirmedCases'] = zeros_column
df_test['Fatalities'] = zeros_column
df_test['maxConfirmedCases'] = zeros_column
class Discriminator(nn.Module):
def __init__(self):
super(Discriminator, self ).__init__()
self.fc1 = nn.Linear(10000, 12288)
 self.conv1 = nn.Conv2d(1, 1,(2, 1), bias=False)
 def forward(self, imgs, imgnames):
x = self.fc1(imgnames)
 x = torch.sigmoid(x)
 x = torch.cat(( imgs, x), dim=1 ).view(-1, 1, 2, 12288)
 x = self.conv1(x)
 return x.view(-1, 12288 )<choose_model_class>
df_train_preserve = df_train.copy()
df_test_preserve = df_test.copy()
lr = 0.005
epochs = 360
netD = Discriminator().to(device)
 optimizerD = optim.Adam(netD.parameters() , lr=lr)
 criteria = nn.BCELoss()
netD.conv1.weight = nn.Parameter(torch.Tensor([[[[ -1.0],
[1.0]]]] ).to(device))
for param in netD.conv1.parameters() :
param.requires_grad = False<compute_test_metric>
def prepare_prediction_template(df_in,withhold=0,extend_to_date = END_PREDICTION_DATE,
verbose=False):

 withhold = abs(withhold)
 if(withhold != 0):
wth_str = str(withhold)+ 'D'
last_date = df_in['DateTime'].max() - pd.to_timedelta(wth_str)
 df_local = df_in[df_in['DateTime']<=last_date].copy()
if(verbose):
print("pptemp withhold={} so last_date to copy was {}".format(withhold,last_date))
else:
df_local = df_in.copy()
if(verbose):
print("pptemp withhold=0 so made complete copy before extension.")
 keep_ps = df_in.iloc[1]['Province_State']
keep_cr = df_in.iloc[1]['Country_Region']
keep_id = df_in.iloc[1]['Id']
keep_sn = df_in.iloc[1]['SimpleName']
max_date = df_local['DateTime'].max()
num_extensions =(pd.to_datetime(extend_to_date)- max_date ).days
if(verbose):
print("Max date was {} so we need {} extensions.".format(max_date,num_extensions))
next_date = max_date + pd.to_timedelta('1D')
 end_date_datetime = pd.to_datetime(extend_to_date)
 while(next_date <= end_date_datetime):
df_local = df_local.append({'Id':0,
'Province_State':keep_ps,
'Country_Region':keep_cr,
'DateTime':next_date,
'ConfirmedCases':0,
'maxConfirmedCases':0,
'Fatalities':0,
'SimpleName':keep_sn},
ignore_index=True)
 next_date = next_date + pd.to_timedelta('1D')
 return df_local,num_extensions
for k in tqdm(range(epochs)) :
for i,(y, X, Zeros)in enumerate(data_loader):
netD.zero_grad()
y_pred = netD(Zeros, X)
 loss = criteria(y_pred, y)
 loss.backward()
optimizerD.step()
if(k + 1)% 2 == 0:
print(f"Epoch: {k+1}/{epochs} | Loss: {loss}" )<normalization>
df_1, num_e = prepare_prediction_template(df_golden,withhold=1,extend_to_date='2020-06-09',verbose=True)
 print("Output num_e={} and df is
{}".format(num_e,df_1))
class Generator(nn.Module):
def __init__(self):
super(Generator, self ).__init__()
self.fc1 = nn.Linear(10000, 12288)
 def forward(self, imgnames):
x = self.fc1(imgnames)
 return x, imgnames.view(-1, 10000 )<data_type_conversions>
mdates = pd.date_range('2020-03-19', '2020-04-30', freq='D')
 date_list = list(mdates.strftime('%Y-%m-%d'))
date_list2 = ['2020-03-19','2020-03-20','2020-03-21','2020-03-22','2020-03-23','2020-03-24','2020-03-25',\
'2020-03-26','2020-03-27',\
'2020-03-28','2020-03-29','2020-03-30','2020-03-31','2020-04-01','2020-04-02','2020-04-03','2020-04-04',\
'2020-04-05','2020-04-06','2020-04-07','2020-04-08','2020-04-09','2020-04-10','2020-04-11','2020-04-12',\
'2020-04-13','2020-04-14','2020-04-15','2020-04-16','2020-04-17','2020-04-18','2020-04-19','2020-04-20',\
'2020-04-21','2020-04-22','2020-04-23','2020-04-24','2020-04-24','2020-04-25','2020-04-26','2020-04-27',\
'2020-04-28','2020-04-29','2020-04-30'
]
def copy_preditions_to_final_list(location, df_predictions, df_final_list,verbose=False):

 if(verbose):
print("Copying predictions for {}:".format(location))
assert(100==df_predictions.shape[0]),'Error 1 - Passed in an incorrect predictions DataFrame'
df_small = df_predictions[df_predictions['SimpleName'] == location]
assert(df_small.shape[0]==df_predictions.shape[0]),'Error 2 - Passed in an incorrect predictions DataFrame'
for date1 in date_list:
date1_dt = pd.to_datetime(date1)
 mask_location = df_final_list['SimpleName'] == location
mask_date = df_final_list['DateTime'] == date1_dt
idx_final = df_final_list[mask_location & mask_date].index[0]
mask_date2 = df_small['DateTime'] == date1_dt
idx_small = df_small[mask_date2].index[0]
if(verbose):
print('idx: final={} small={}'.format(idx_final, idx_small))
df_final_list.at[idx_final,'ConfirmedCases'] = df_small.at[idx_small,'ConfirmedCases']
df_final_list.at[idx_final,'maxConfirmedCases'] = df_small.at[idx_small,'maxConfirmedCases']
df_final_list.at[idx_final,'Fatalities'] = df_small.at[idx_small,'Fatalities']
def show_image() :
plt.figure(figsize=(15,3))
for j in range(5):
with torch.no_grad() :
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 inp = torch.Tensor(xx.reshape(( -1,10000)) ).to(device)
 img = netG(inp)[0].reshape(( -1,64,64,3)).to("cpu" ).clone().detach().numpy()
img = Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()
def show_d_images(imgs, title):
with torch.no_grad() :
imgs = imgs.reshape(( -1,64,64,3))
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = imgs[j].detach().cpu().numpy()
img = Image.fromarray(( 255*img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.title(title)
 plt.imshow(img)
 plt.show()<choose_model_class>
def single_model(location, practice_withhold=0, verbose=False):
this_name = location
df_sname = df_train[df_train['SimpleName'] == this_name].copy()
df_sname['maxConfirmedCases'] = 0
df_sname_template,num_extended = prepare_prediction_template(df_sname,(-1*practice_withhold),verbose=verbose)
 df_sname.head()
this_beta, this_gamma, this_ifx, this_rfx, this_pop = db_parameters.loc[this_name]
df_SIR_predictions = sir_model(df_sname_template,N=this_pop,\
beta=this_beta,gamma=this_gamma,\
i_factor = this_ifx, r_factor = this_rfx)
 if(verbose):
starter = practice_withhold + num_extended
print("ends to compare are
{}
***AND***
{}".format(df_sname,
df_SIR_predictions.iloc[:(-1*num_extended)]))
rmsle_c, rmsle_f = evaluate_predictions(df_sname,df_SIR_predictions,
practice_withhold,num_extended,
'simple SIR',verbose)
 rmsle_c2, rmsle_f2 = evaluate_predictions2(df_sname,df_SIR_predictions,
practice_withhold,num_extended,
'simple SIR',verbose)
 plot_title = this_name + " v1 SIR plots"
print("Finished evaluating...{}: {} {}".format(this_name,rmsle_c,rmsle_f))
print("Finished evaluating 2...{}: {} {}".format(this_name,rmsle_c2,rmsle_f2))
copy_preditions_to_final_list(this_name, df_SIR_predictions, df_test)
 prediction_scores[this_name] = [rmsle_c2, rmsle_f2]
return df_SIR_predictions, df_sname, plot_title
lr = 0.01
beta1 = 0.5
netG = Generator().to(device)
 optimizerG = optim.Adam(netG.parameters() , lr=lr)
 criterion = nn.MSELoss()
netD.conv1.weight = nn.Parameter(torch.Tensor([[[[ -1.],
[1.]]]] ).to(device))
for param in netD.parameters() :
param.requires_grad = False<compute_train_metric>
practice_withhold = 7
verbose = True
this_name = 'Hubei__China'
df_sname = df_train[df_train['SimpleName'] == this_name].copy()
df_sname['maxConfirmedCases'] = 0
df_sname_template,num_extended = prepare_prediction_template(df_sname,(-1*practice_withhold),verbose=verbose)
 df_sname.head()
this_beta, this_gamma, this_ifx, this_rfx, this_pop = db_parameters.loc[this_name]
df_SIR_predictions = sir_model(df_sname_template,N=this_pop,\
beta=this_beta,gamma=this_gamma,\
i_factor = this_ifx, r_factor = this_rfx )
epochs = 50
for epoch in tqdm(range(epochs)) :
for i,(y, X, Zeros)in enumerate(data_loader):
netG.zero_grad()
fake, seed = netG(X)
 y_pred = netD(Zeros, seed)
 errG = criterion(fake, y_pred)
 errG.backward()
optimizerG.step()
if(epoch+1)% 5 == 0:
print(f"Epoch: {epoch+1}: G_Loss: {errG}")
 show_d_images(fake, "Generator output" )<set_options>
rmsle_c, rmsle_f = evaluate_predictions(df_sname,df_SIR_predictions,
practice_withhold,num_extended,
'simple SIR',verbose)
 rmsle_c2, rmsle_f2 = evaluate_predictions2(df_sname,df_SIR_predictions,
practice_withhold,num_extended,
'simple SIR',verbose)
 plot_title = this_name + " v1 SIR plots"
print("Finished evaluating...{}: {} {}".format(this_name,rmsle_c,rmsle_f))
print("Finished evaluating 2...{}: {} {}".format(this_name,rmsle_c2,rmsle_f2))
class DogGenerator() :
index = 0
t = [
transforms.RandomCrop(( 48, 48), padding=None, pad_if_needed=True, fill=0, padding_mode='symmetric'),
transforms.Scale(( 64,64))
]
tfms = transforms.Compose([
transforms.RandomHorizontalFlip(p=0.5),
transforms.ColorJitter(brightness=(1,1.3), contrast=(1,1.3), saturation=0, hue=0)
 ])
 def getDog(self,seed):
xx = torch.Tensor(np.zeros(( 10000)) ).to(device)
 xx[self.index] = 0.999999
xx[np.random.randint(10000)] = 0.000001
img = netG(xx.reshape(( -1,10000)))[0].reshape(( 64,64,3)).detach().cpu().numpy() * 255
self.index =(self.index+1)%10000
return self.tfms(Image.fromarray(img.astype('uint8')) )<import_modules>
df_sname = df_train[df_train['SimpleName'] == 'not_provided__Germany']
batch_size = 16<define_variables>
df_sname_template_1,num_extended = prepare_prediction_template(df_sname,-5 )
start = time()<set_options>
df_SIR_practice = sir_model(df_sname_template_1 )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<load_pretrained>
selected_country_region = "US"
selected_province_state = "Washington"
sname = selected_province_state + '__' + selected_country_region
print("Sanity check for {}".format(sname))
class DataGenerator(Dataset):
def __init__(self, directory, transform=None, n_samples=np.inf):
self.directory = directory
self.transform = transform
self.n_samples = n_samples
self.samples = self._load_subfolders_images(directory)
 if len(self.samples)== 0:
raise RuntimeError("Found 0 files in subfolders of: {}".format(directory))
def _load_subfolders_images(self, root):
IMG_EXTENSIONS =(
 '.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
 def is_valid_file(x):
return torchvision.datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 required_transforms = torchvision.transforms.Compose([
torchvision.transforms.Resize(64),
torchvision.transforms.CenterCrop(64),
])
 imgs = []
paths = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames)[:min(self.n_samples, 999999999999999)]:
path = os.path.join(root, fname)
 paths.append(path)
 for path in paths:
if is_valid_file(path):
img = dset.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(
 dirname for dirname in os.listdir('.. /input/annotation/Annotation/')if
dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join('.. /input/annotation/Annotation/',
annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
bbox =(xmin, ymin, xmin+w, ymin+w)
 object_img = required_transforms(img.crop(bbox))
imgs.append(object_img)
 return imgs
def __getitem__(self, index):
sample = self.samples[index]
if self.transform is not None:
sample = self.transform(sample)
 return np.asarray(sample)
 def __len__(self):
return len(self.samples)
<categorify>
db_parameters.loc['Isle of Man__United Kingdom']
%%time
database = '.. /input/all-dogs/all-dogs/'
transform = transforms.Compose([transforms.RandomHorizontalFlip(p=0.1),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 train_data = DataGenerator(database, transform=transform,n_samples=25000)
 train_loader = torch.utils.data.DataLoader(train_data, shuffle=True,batch_size=batch_size, num_workers = 4)
<normalization>
sname_set
mini_set = ['not_provided__Togo',
'not_provided__Cambodia',
'Connecticut__US',
'Jilin__China',
'Greenland__Denmark',
'Maine__US',
'Pennsylvania__US',
'Quebec__Canada',
'New Jersey__US',
'Manitoba__Canada',
'not_provided__Haiti']
class PixelwiseNorm(nn.Module):
def __init__(self):
super(PixelwiseNorm, self ).__init__()
def forward(self, x, alpha=1e-8):

 y = x.pow(2.).mean(dim=1, keepdim=True ).add(alpha ).sqrt()
y = x / y
return y<categorify>
df_train = df_train_preserve.copy()
df_test = df_test_preserve.copy()
class MinibatchStdDev(th.nn.Module):

 def __init__(self):

 super(MinibatchStdDev, self ).__init__()
def forward(self, x, alpha=1e-8):

 batch_size, _, height, width = x.shape
y = x - x.mean(dim=0, keepdim=True)
 y = th.sqrt(y.pow(2.).mean(dim=0, keepdim=False)+ alpha)
 y = y.mean().view(1, 1, 1, 1)
 y = y.repeat(batch_size,1, height, width)
 y = th.cat([x, y], 1)
 return y 
<define_search_model>
last_day = df_test[df_test['DateTime'] == pd.to_datetime('2020-04-30')]
print("Final prediction for 2020-04-30 has {} ConfirmedCases and {} Fatalities".format(last_day['ConfirmedCases'].sum() ,\
last_day['Fatalities'].sum()))
class Generator(nn.Module):
def __init__(self, nz, nfeats, nchannels):
super(Generator, self ).__init__()
self.conv1 = spectral_norm(nn.ConvTranspose2d(nz, nfeats * 8, 4, 1, 0, bias=False))
self.conv2 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 8, 4, 2, 1, bias=False))
self.conv3 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 4, 4, 2, 1, bias=False))
self.conv4 = spectral_norm(nn.ConvTranspose2d(nfeats * 4, nfeats * 2, 4, 2, 1, bias=False))
self.conv5 = spectral_norm(nn.ConvTranspose2d(nfeats * 2, nfeats, 4, 2, 1, bias=False))
self.conv6 = spectral_norm(nn.ConvTranspose2d(nfeats, nchannels, 3, 1, 1, bias=False))
self.pixnorm = PixelwiseNorm()
def forward(self, x):
x = F.leaky_relu(self.conv1(x))
x = F.leaky_relu(self.conv2(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv3(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv4(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv5(x))
x = self.pixnorm(x)
 x = torch.tanh(self.conv6(x))
return x
class Discriminator(nn.Module):
def __init__(self, nchannels, nfeats):
super(Discriminator, self ).__init__()
self.conv1 = nn.Conv2d(nchannels, nfeats, 4, 2, 1, bias=False)
 self.conv2 = spectral_norm(nn.Conv2d(nfeats, nfeats * 2, 4, 2, 1, bias=False))
self.bn2 = nn.BatchNorm2d(nfeats * 2)
 self.conv3 = spectral_norm(nn.Conv2d(nfeats * 2, nfeats * 4, 4, 2, 1, bias=False))
self.bn3 = nn.BatchNorm2d(nfeats * 4)
 self.conv4 = spectral_norm(nn.Conv2d(nfeats * 4, nfeats * 8, 4, 2, 1, bias=False))
self.bn4 = nn.MaxPool2d(2)
 self.batch_discriminator = MinibatchStdDev()
self.conv5 = spectral_norm(nn.Conv2d(nfeats * 8 +1, 1, 2, 1, 0, bias=False))
def forward(self, x):
x = F.leaky_relu(self.conv1(x), 0.2)
 x = F.leaky_relu(self.bn2(self.conv2(x)) , 0.2)
 x = F.leaky_relu(self.bn3(self.conv3(x)) , 0.2)
 x = F.leaky_relu(self.bn4(self.conv4(x)) , 0.2)
 x = self.batch_discriminator(x)
 x = torch.sigmoid(self.conv5(x))
return x.view(-1, 1)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 lr = 0.0003
lr_d = 0.0001
beta1 = 0.5
epochs = 2000
netG = Generator(100, 32, 3 ).to(device)
 netD = Discriminator(3, 48 ).to(device)
 criterion = nn.BCELoss()
optimizerD = optim.Adam(netD.parameters() , lr=lr, betas=(beta1, 0.999))
optimizerG = optim.Adam(netG.parameters() , lr=lr_d, betas=(beta1, 0.999))
lr_schedulerG = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerG,
T_0=epochs//200, eta_min=0.00005)
 lr_schedulerD = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerD,
T_0=epochs//200, eta_min=0.00005)
 nz = 100
fixed_noise = torch.randn(25, nz, 1, 1, device=device)
 real_label = 0.8
fake_label = 0.0
batch_size = train_loader.batch_size
step = 0
for epoch in range(epochs):
for ii,(real_images)in tqdm(enumerate(train_loader), total=len(train_loader)) :
end = time()
if(end -start)> 30000 :
break
netD.zero_grad()
real_images = real_images.to(device)
 batch_size = real_images.size(0)
 labels = torch.full(( batch_size, 1), real_label, device=device)+ np.random.uniform(-0.1, 0.1)
 output = netD(real_images)
 errD_real = criterion(output, labels)
 errD_real.backward()
D_x = output.mean().item()
noise = torch.randn(batch_size, nz, 1, 1, device=device)
 fake = netG(noise)
 labels.fill_(fake_label)+ np.random.uniform(0, 0.2)
 output = netD(fake.detach())
 errD_fake = criterion(output, labels)
 errD_fake.backward()
D_G_z1 = output.mean().item()
errD = errD_real + errD_fake
optimizerD.step()
netG.zero_grad()
labels.fill_(real_label)
 output = netD(fake)
 errG = criterion(output, labels)
 errG.backward()
D_G_z2 = output.mean().item()
optimizerG.step()
if step % 500 == 0:
print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)) : %.4f / %.4f'
%(epoch + 1, epochs, ii, len(train_loader),
errD.item() , errG.item() , D_x, D_G_z1, D_G_z2))
valid_image = netG(fixed_noise)
 step += 1
lr_schedulerG.step(epoch)
 lr_schedulerD.step(epoch)
 if epoch % 5 == 0:
show_generated_img()
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values
if not os.path.exists('.. /output_images'):
os.mkdir('.. /output_images')
 im_batch_size = 50
n_images=10000
for i_batch in range(0, n_images, im_batch_size):
z = truncated_normal(( im_batch_size, 100, 1, 1), threshold=1)
 gen_z = torch.from_numpy(z ).float().to(device)
 gen_images = netG(gen_z)
 images = gen_images.to("cpu" ).clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
 for i_image in range(gen_images.size(0)) :
save_image(( gen_images[i_image, :, :, :] +1.0)/2.0, os.path.join('.. /output_images', f'image_{i_batch+i_image:05d}.png'))
shutil.make_archive('images', 'zip', '.. /output_images' )<load_from_csv>
df_scores = pd.DataFrame.from_dict(prediction_scores,orient='index', columns=['cc_score','f_score'] )
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 df.shape, test.shape<concatenate>
df_sub = df_test.drop(['Province_State','SimpleName','Country_Region','Date','DateTime'],axis=1 )
df_all = df.append(test)
 df_all.shape<count_missing_values>
df_sub_mod = df_sub.drop('ConfirmedCases', axis=1 )
df['Target'].isnull().sum()<count_values>
df_sub_mod.columns = ['ForecastId','Fatalities','ConfirmedCases']
df_sub_mod.head()
df_all['Target'].value_counts()<count_values>
df_sub = df_sub_mod
df.dtypes.value_counts()<count_values>
df_sub.to_csv('submission.csv',index=False)