Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<train_model>
%matplotlib inline
print(os.listdir(".. /input"))
clf.fit(train,y)
 ans = clf.predict_proba(test)
 ans<load_from_csv>
seed = 2019
random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 BATCH_SIZE = 32
NUM_WORKERS = 4
EMA = False
LABEL_NOISE = False
LABEL_NOISE_PROB = 0.1
submit = pd.read_csv('.. /input/sample_submission.csv')
 submit.columns<feature_engineering>
PATH = '.. /input/all-dogs/all-dogs/'
img_filenames = os.listdir(PATH)
 len(img_filenames )
submit['target'] = ans[:,1]<save_to_csv>
PATH_ANNOTATION = '.. /input/annotation/Annotation/'
breeds = glob.glob(PATH_ANNOTATION+'*')
 annotations = []
for breed in breeds:
annotations += glob.glob(breed+'/*')
 len(annotations )
submit.to_csv('submit.csv', index = False )<import_modules>
breed_map = {}
for annotation in annotations:
breed = annotation.split('/')[-2]
index = breed.split('-')[0]
breed_map.setdefault(index,breed)
 n_classes = len(breed_map)
 n_classes
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score
from sklearn.linear_model import LinearRegression,LogisticRegression
from sklearn import model_selection<load_from_csv>
intruders = [
'n02088238_10870_0.jpg',
'n02088466_6901_1.jpg',
'n02088466_6963_0.jpg',
'n02088466_9167_0.jpg',
'n02088466_9167_1.jpg',
'n02088466_9167_2.jpg',
'n02089867_2221_0.jpg',
'n02089867_2227_1.jpg',
'n02089973_1132_3.jpg',
'n02089973_1352_3.jpg',
'n02089973_1458_1.jpg',
'n02089973_1799_2.jpg',
'n02089973_2791_3.jpg',
'n02089973_4055_0.jpg',
'n02089973_4185_1.jpg',
'n02089973_4185_2.jpg',
'n02090379_4673_1.jpg',
'n02090379_4875_1.jpg',
'n02090622_7705_1.jpg',
'n02090622_9358_1.jpg',
'n02090622_9883_1.jpg',
'n02090721_209_1.jpg',
'n02090721_1222_1.jpg',
'n02090721_1534_1.jpg',
'n02090721_1835_1.jpg',
'n02090721_3999_1.jpg',
'n02090721_4089_1.jpg',
'n02090721_4276_2.jpg',
'n02091032_722_1.jpg',
'n02091032_745_1.jpg',
'n02091032_1773_0.jpg',
'n02091032_9592_0.jpg',
'n02091134_2349_1.jpg',
'n02091134_14246_2.jpg',
'n02091244_583_1.jpg',
'n02091244_2407_0.jpg',
'n02091244_3438_1.jpg',
'n02091244_5639_1.jpg',
'n02091244_5639_2.jpg',
'n02091467_473_0.jpg',
'n02091467_4386_1.jpg',
'n02091467_4427_1.jpg',
'n02091467_4558_1.jpg',
'n02091467_4560_1.jpg',
'n02091635_1192_1.jpg',
'n02091635_4422_0.jpg',
'n02091831_1594_1.jpg',
'n02091831_2880_0.jpg',
'n02091831_7237_1.jpg',
'n02092002_1551_1.jpg',
'n02092002_1937_1.jpg',
'n02092002_4218_0.jpg',
'n02092002_4596_0.jpg',
'n02092002_5246_1.jpg',
'n02092002_6518_0.jpg',
'n02093256_1826_1.jpg',
'n02093256_4997_0.jpg',
'n02093256_14914_0.jpg',
'n02093428_5662_0.jpg',
'n02093428_6949_1.jpg'
]
len(intruders )
train = pd.read_csv(".. /input/train.csv")
 print(train.shape)
 test = pd.read_csv(".. /input/test.csv")
 print(test.shape )<prepare_x_and_y>
%%time
breed_map_2 = {}
for i,b in enumerate(breed_map.keys()):
breed_map_2[b] = i
train_x = train.iloc[:,2:]
train_y_ = train.iloc[:,1]
train_y = []
for i in train_y_:
train_y.append(i)
 test_x = test.iloc[:,1:] 
<normalization>
class DogDataset(Dataset):
def __init__(self, path, img_list, transform1=None, transform2=None):
self.path = path
self.img_list = img_list
self.transform1 = transform1
self.transform2 = transform2
self.imgs = []
self.labels = []
for i,full_img_path in enumerate(self.img_list):
if full_img_path in intruders:
continue
img_path = full_img_path[:-6]+'.jpg'
bbx_idx = int(full_img_path[-5])
 img = data_preprocessing(img_path,bbx_idx)
 if self.transform1:
img = self.transform1(img)
 self.imgs.append(img)
 label = breed_map_2[img_path.split('_')[0]]
self.labels.append(label)
 def __len__(self):
return len(self.imgs)
 def __getitem__(self,idx):
img = self.imgs[idx]
if self.transform2:
img = self.transform2(img)
 label = self.labels[idx]
return {'img':img, 'label':label}
scaler = StandardScaler()
train_x = scaler.fit_transform(train_x)
<find_best_model_class>
%%time
img_size = 64
batch_size = BATCH_SIZE
MEAN1,MEAN2,MEAN3 = 0.5, 0.5, 0.5
STD1,STD2,STD3 = 0.5, 0.5, 0.5
transform1 = transforms.Compose([transforms.Resize(img_size)])
 transform2 = transforms.Compose([transforms.RandomCrop(img_size),
transforms.RandomHorizontalFlip(p=0.5),
transforms.ToTensor() ,
transforms.Normalize(mean=[MEAN1, MEAN2, MEAN3],
std=[STD1, STD2, STD3]),
])
 train_set = DogDataset(path=PATH,
img_list=img_filenames_th,
transform1=transform1,
transform2=transform2, 
)
 train_loader = DataLoader(train_set,
shuffle=True, batch_size=batch_size,
num_workers=NUM_WORKERS, pin_memory=True )
regression = LogisticRegression(class_weight='balanced', solver='liblinear', penalty ='l1', C= 0.1, max_iter=10000)
 regression.fit(train_x,train_y)
 scores = model_selection.cross_val_score(regression,train_x,train_y,scoring="accuracy",cv=50)
 test_y_ = regression.predict_proba(test_x)
 print("Training Accuracy score: ",regression.score(train_x,train_y))
test_y = []
print(( test_y_[:,1])) 
<save_to_csv>
img = data_preprocessing(img_filenames_th[1500][:-6]+'.jpg',0)
 img = transform1(img)
 img
submission = pd.DataFrame({"id":test["id"],"target":test_y_[:,1]})
 submission.head()
filename = 'submission.csv'
submission.to_csv(filename,index=False)
 print('Saved file: ' + filename )<set_options>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 device
!pip install scikit-misc
warnings.filterwarnings("ignore")
 invlogit = lambda x: 1/(1 + tt.exp(-x))
def trace_predict(trace, X):
y_hat = np.apply_along_axis(np.mean, 1, expit(trace['alpha'] + np.dot(X, np.transpose(trace['beta']))))
return(y_hat)
 def posterior_predict(trace, model, n=1000, progressbar=True):
with model:
ppc = pm.sample_posterior_predictive(trace,n, progressbar=progressbar)
 return(np.mean(np.array(ppc['y_obs']), axis=0))
def gather(df, key, value, cols):
id_vars = [ col for col in df.columns if col not in cols ]
id_values = cols
var_name = key
value_name = value
return pd.melt(df, id_vars, id_values, var_name, value_name)
 def spread(df, index, columns, values):
return df.pivot(index, columns, values ).reset_index(level=index ).rename_axis(None,axis=1)
 def fit_loess(df, transform_logit=False):
l = loess(df["value"],df["target"])
 l.fit()
pred_obj = l.predict(df["value"],stderror=True)
 conf = pred_obj.confidence()
yhat = pred_obj.values
ll = conf.lower
ul = conf.upper
df["loess"] = np.clip(yhat,.001,.999)
 df["ll"] = np.clip(ll,.001,.999)
 df["ul"] = np.clip(ul,.001,.999)
 if transform_logit:
df["logit_loess"] = logit(df["loess"])
 df["logit_ll"] = logit(df["ll"])
 df["logit_ul"] = logit(df["ul"])
 return(df)
 def plot_loess(df, features):
z = gather(df[["id","target"]+features], "feature", "value", features)
 z = z.groupby("feature" ).apply(fit_loess, transform_logit=True)
 z["feature"] = pd.to_numeric(z["feature"])
 plot =(
 ggplot(z, aes("value","logit_loess",ymin="logit_ll",ymax="logit_ul")) +
geom_point(alpha=.5)+
geom_line(alpha=.5)+
geom_ribbon(alpha=.33)+
facet_wrap("~feature")
)
 return(plot)
 df = pd.read_csv(".. /input/train.csv")
 y = np.asarray(df.target)
 X = np.array(df.ix[:, 2:302])
 df2 = pd.read_csv('.. /input/test.csv')
 df2.head()
X2 = np.array(df2.ix[:, 1:301])
 print("training shape: ", X.shape)
 print("test shape: ", X2.shape )<train_on_grid>
def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad )
def make_model(X, y, cauchy_scale):
model = pm.Model()
with model:
alpha = pm.Normal('alpha', mu=0, sd=3)
 beta = pm.Cauchy('beta', alpha=0, beta=cauchy_scale, shape=X.shape[1])
 mu = pm.math.dot(X, beta)
 y_obs = pm.Bernoulli('y_obs', p=invlogit(alpha + mu), observed=y)
 model.name = "linear_c_"+str(cauchy_scale)
 return(model)
<compute_train_metric>
def conv3x3(in_channel, out_channel):
return nn.Conv2d(in_channel,out_channel,
kernel_size=3,stride=1,padding=1,dilation=1,bias=False)
 def conv1x1(in_channel, out_channel):
return nn.Conv2d(in_channel,out_channel,
kernel_size=1,stride=1,padding=0,dilation=1,bias=False)
 def init_weight(m):
classname = m.__class__.__name__
if classname.find('Conv')!= -1:
nn.init.orthogonal_(m.weight, gain=1)
 if m.bias is not None:
m.bias.data.zero_()
elif classname.find('Batch')!= -1:
m.weight.data.normal_(1,0.02)
 m.bias.data.zero_()
elif classname.find('Linear')!= -1:
nn.init.orthogonal_(m.weight, gain=1)
 if m.bias is not None:
m.bias.data.zero_()
elif classname.find('Embedding')!= -1:
nn.init.orthogonal_(m.weight, gain=1)
 class Attention(nn.Module):
def __init__(self, channels):
super().__init__()
self.channels = channels
self.theta = nn.utils.spectral_norm(conv1x1(channels, channels//8)).apply(init_weight)
 self.phi = nn.utils.spectral_norm(conv1x1(channels, channels//8)).apply(init_weight)
 self.g = nn.utils.spectral_norm(conv1x1(channels, channels//2)).apply(init_weight)
 self.o = nn.utils.spectral_norm(conv1x1(channels//2, channels)).apply(init_weight)
 self.gamma = nn.Parameter(torch.tensor(0.) , requires_grad=True)
 def forward(self, inputs):
batch,c,h,w = inputs.size()
theta = self.theta(inputs)
 phi = F.max_pool2d(self.phi(inputs), [2,2])
 g = F.max_pool2d(self.g(inputs), [2,2])
 theta = theta.view(batch, self.channels//8, -1)
 phi = phi.view(batch, self.channels//8, -1)
 g = g.view(batch, self.channels//2, -1)
 beta = F.softmax(torch.bmm(theta.transpose(1,2), phi), -1)
 o = self.o(torch.bmm(g, beta.transpose(1,2)).view(batch,self.channels//2,h,w))
return self.gamma*o + inputs
class ConditionalNorm(nn.Module):
def __init__(self, in_channel, n_condition):
super().__init__()
self.bn = nn.BatchNorm2d(in_channel, affine=False)
 self.embed = nn.Linear(n_condition, in_channel* 2)
 nn.init.orthogonal_(self.embed.weight.data[:, :in_channel], gain=1)
 self.embed.weight.data[:, in_channel:].zero_()
def forward(self, inputs, label):
out = self.bn(inputs)
 embed = self.embed(label.float())
 gamma, beta = embed.chunk(2, dim=1)
 gamma = gamma.unsqueeze(2 ).unsqueeze(3)
 beta = beta.unsqueeze(2 ).unsqueeze(3)
 out = gamma * out + beta
return out
comp = pm.stats.compare(model_dict, ic="LOO", method='BB-pseudo-BMA')
 for i in range(0,len(traces)) :
y_hat = trace_predict(traces[i], X)
 print("scale = ",cauchy_scale_pars[i],", training AUCROC:",roc_auc_score(y,y_hat))
comp<find_best_params>
model1 = models[5]
trace1 = traces[5]
model1.name<save_to_csv>
class ResBlock_G(nn.Module):
def __init__(self, in_channel, out_channel, condition_dim, upsample=True):
super().__init__()
self.cbn1 = ConditionalNorm(in_channel, condition_dim)
 self.upsample = nn.Sequential()
if upsample:
self.upsample.add_module('upsample',nn.Upsample(scale_factor=2, mode='nearest'))
self.conv3x3_1 = nn.utils.spectral_norm(conv3x3(in_channel, out_channel)).apply(init_weight)
 self.cbn2 = ConditionalNorm(out_channel, condition_dim)
 self.conv3x3_2 = nn.utils.spectral_norm(conv3x3(out_channel, out_channel)).apply(init_weight)
 self.conv1x1 = nn.utils.spectral_norm(conv1x1(in_channel, out_channel)).apply(init_weight)
 def forward(self, inputs, condition):
x = F.leaky_relu(self.cbn1(inputs, condition))
x = self.upsample(x)
 x = self.conv3x3_1(x)
 x = self.conv3x3_2(F.leaky_relu(self.cbn2(x, condition)))
 x += self.conv1x1(self.upsample(inputs))
return x
class Generator(nn.Module):
def __init__(self, n_feat, codes_dim=24, n_classes=n_classes):
super().__init__()
self.fc = nn.Sequential(
 nn.utils.spectral_norm(nn.Linear(codes_dim, 16*n_feat*4*4)).apply(init_weight)
)
 self.res1 = ResBlock_G(16*n_feat, 16*n_feat, codes_dim+n_classes, upsample=True)
 self.res2 = ResBlock_G(16*n_feat, 8*n_feat, codes_dim+n_classes, upsample=True)
 self.res3 = ResBlock_G(8*n_feat, 4*n_feat, codes_dim+n_classes, upsample=True)
 self.attn = Attention(4*n_feat)
 self.res4 = ResBlock_G(4*n_feat, 2*n_feat, codes_dim+n_classes, upsample=True)
 self.conv = nn.Sequential(
 nn.LeakyReLU() ,
nn.utils.spectral_norm(conv3x3(2*n_feat, 3)).apply(init_weight), 
)
 def forward(self, z, label_ohe, codes_dim=24):

 batch = z.size(0)
 z = z.squeeze()
label_ohe = label_ohe.squeeze()
codes = torch.split(z, codes_dim, dim=1)
 x = self.fc(codes[0])
 x = x.view(batch,-1,4,4)
 condition = torch.cat([codes[1], label_ohe], dim=1)
 x = self.res1(x, condition)
 condition = torch.cat([codes[2], label_ohe], dim=1)
 x = self.res2(x, condition)
 condition = torch.cat([codes[3], label_ohe], dim=1)
 x = self.res3(x, condition)
 x = self.attn(x)
 condition = torch.cat([codes[4], label_ohe], dim=1)
 x = self.res4(x, condition)
 x = self.conv(x)
 x = torch.tanh(x)
 return x
class ResBlock_D(nn.Module):
def __init__(self, in_channel, out_channel, downsample=True):
super().__init__()
self.layer = nn.Sequential(
 nn.LeakyReLU(0.2),
nn.utils.spectral_norm(conv3x3(in_channel, out_channel)).apply(init_weight),
nn.LeakyReLU(0.2),
nn.utils.spectral_norm(conv3x3(out_channel, out_channel)).apply(init_weight), 
)
 self.shortcut = nn.Sequential(
 nn.utils.spectral_norm(conv1x1(in_channel,out_channel)).apply(init_weight), 
)
 if downsample:
self.layer.add_module('avgpool',nn.AvgPool2d(kernel_size=2,stride=2))
self.shortcut.add_module('avgpool',nn.AvgPool2d(kernel_size=2,stride=2))
def forward(self, inputs):
x = self.layer(inputs)
 x += self.shortcut(inputs)
 return x
class Discriminator(nn.Module):
def __init__(self, n_feat, n_classes=n_classes):
super().__init__()
self.res1 = ResBlock_D(3, n_feat, downsample=True)
 self.attn = Attention(n_feat)
 self.res2 = ResBlock_D(n_feat, 2*n_feat, downsample=True)
 self.res3 = ResBlock_D(2*n_feat, 4*n_feat, downsample=True)
 self.res4 = ResBlock_D(4*n_feat, 8*n_feat, downsample=True)
 self.res5 = ResBlock_D(8*n_feat,16*n_feat, downsample=False)
 self.fc = nn.utils.spectral_norm(nn.Linear(16*n_feat,1)).apply(init_weight)
 self.embedding = nn.Embedding(num_embeddings=n_classes, embedding_dim=16*n_feat ).apply(init_weight)
 def forward(self, inputs, label):
batch = inputs.size(0)
 h = self.res1(inputs)
 h = self.attn(h)
 h = self.res2(h)
 h = self.res3(h)
 h = self.res4(h)
 h = self.res5(h)
 h = torch.sum(( F.leaky_relu(h,0.2)).view(batch,-1,4*4), dim=2)
 outputs = self.fc(h)
 if label is not None:
embed = self.embedding(label)
 outputs += torch.sum(embed*h,dim=1,keepdim=True)
 outputs = torch.sigmoid(outputs)
 return outputs
def generate_submission(trace, file_suffix=""):
test_predictions = trace_predict(trace, X2)
 submission = pd.DataFrame({'id':df2.id, 'target':test_predictions})
 submission.to_csv("submission_"+file_suffix+".csv", index = False)
 return(None)
 for model in model_dict.keys() :
generate_submission(model_dict[model], model.name )<create_dataframe>
def generate_img(netG,fixed_noise,fixed_aux_labels=None):
if fixed_aux_labels is not None:
gen_image = netG(fixed_noise,fixed_aux_labels ).to('cpu' ).clone().detach().squeeze(0)
 else:
gen_image = netG(fixed_noise ).to('cpu' ).clone().detach().squeeze(0)
 gen_image = gen_image*0.5 + 0.5
gen_image_numpy = gen_image.numpy().transpose(0,2,3,1)
 return gen_image_numpy
def show_generate_imgs(netG,fixed_noise,fixed_aux_labels=None):
gen_images_numpy = generate_img(netG,fixed_noise,fixed_aux_labels)
 fig = plt.figure(figsize=(25, 16))
for i, img in enumerate(gen_images_numpy):
ax = fig.add_subplot(4, 8, i + 1, xticks=[], yticks=[])
 plt.imshow(img)
 plt.show()
plt.close()
comp_MA = pm.stats.compare(dict(zip(models[0:-3], traces[0:-3])) , ic="LOO", method='BB-pseudo-BMA')
 ppc_w = pm.sample_posterior_predictive_w(traces[0:-3], 4000, [make_model(X2,np.zeros(19750),c)for c in cauchy_scale_pars[0:-3]],
weights=comp_MA.weight.sort_index(ascending=True),
progressbar=True)
 y_hatMA = np.mean(np.array(ppc_w['y_obs']), axis=0)
 submission = pd.DataFrame({'id':df2.id, 'target':y_hatMA})
 submission.to_csv('submission_MA.csv', index = False )<compute_train_metric>
def run(lr_G=3e-4,lr_D=3e-4, beta1=0.0, beta2=0.999, nz=120, epochs=2,
n_ite_D=1, ema_decay_rate=0.999, show_epoch_list=None, output_freq=10):
netG = Generator(n_feat=36, codes_dim=24, n_classes=n_classes ).to(device)
 netD = Discriminator(n_feat=42, n_classes=n_classes ).to(device)
 if EMA:
netG_EMA = Generator(n_feat=42, codes_dim=24, n_classes=n_classes ).to(device)
 netG_EMA.load_state_dict(netG.state_dict())
 for p in netG_EMA.parameters() :
p.requires_grad = False
print(count_parameters(netG))
print(count_parameters(netD))
real_label = 0.9
fake_label = 0
D_loss_list = []
G_loss_list = []
dis_criterion = nn.BCELoss().to(device)
 optimizerD = optim.Adam(netD.parameters() , lr=lr_D, betas=(beta1, beta2))
optimizerG = optim.Adam(netG.parameters() , lr=lr_G, betas=(beta1, beta2))
fixed_noise = torch.randn(32, nz, 1, 1, device=device)
 fixed_aux_labels = np.random.randint(0,n_classes, 32)
 fixed_aux_labels_ohe = np.eye(n_classes)[fixed_aux_labels]
fixed_aux_labels_ohe = torch.from_numpy(fixed_aux_labels_ohe[:,:,np.newaxis,np.newaxis])
 fixed_aux_labels_ohe = fixed_aux_labels_ohe.float().to(device, non_blocking=True)
 netG.train()
netD.train()
for epoch in range(1,epochs+1):
if elapsed_time(start_time)> TIME_LIMIT:
print(f'elapsed_time go beyond {TIME_LIMIT} sec')
 break
D_running_loss = 0
G_running_loss = 0
for ii, data in enumerate(train_loader):
for _ in range(n_ite_D):
if LABEL_NOISE:
real_label = 0.9
fake_label = 0
if np.random.random() < LABEL_NOISE_PROB:
real_label = 0
fake_label = 0.9
netD.zero_grad()
real_images = data['img'].to(device, non_blocking=True)
 batch_size = real_images.size(0)
 dis_labels = torch.full(( batch_size, 1), real_label, device=device)
 aux_labels = data['label'].long().to(device, non_blocking=True)
 dis_output = netD(real_images, aux_labels)
 errD_real = dis_criterion(dis_output, dis_labels)
 errD_real.backward(retain_graph=True)
 noise = torch.randn(batch_size, nz, 1, 1, device=device)
 aux_labels = np.random.randint(0,n_classes, batch_size)
 aux_labels_ohe = np.eye(n_classes)[aux_labels]
aux_labels_ohe = torch.from_numpy(aux_labels_ohe[:,:,np.newaxis,np.newaxis])
 aux_labels_ohe = aux_labels_ohe.float().to(device, non_blocking=True)
 aux_labels = torch.from_numpy(aux_labels ).long().to(device, non_blocking=True)
 fake = netG(noise, aux_labels_ohe)
 dis_labels.fill_(fake_label)
 dis_output = netD(fake.detach() ,aux_labels)
 errD_fake = dis_criterion(dis_output, dis_labels)
 errD_fake.backward(retain_graph=True)
 D_running_loss +=(errD_real.item() + errD_fake.item())/len(train_loader)
 optimizerD.step()
netG.zero_grad()
dis_labels.fill_(real_label)
 noise = torch.randn(batch_size, nz, 1, 1, device=device)
 aux_labels = np.random.randint(0,n_classes, batch_size)
 aux_labels_ohe = np.eye(n_classes)[aux_labels]
aux_labels_ohe = torch.from_numpy(aux_labels_ohe[:,:,np.newaxis,np.newaxis])
 aux_labels_ohe = aux_labels_ohe.float().to(device, non_blocking=True)
 aux_labels = torch.from_numpy(aux_labels ).long().to(device, non_blocking=True)
 fake = netG(noise, aux_labels_ohe)
 dis_output = netD(fake, aux_labels)
 errG = dis_criterion(dis_output, dis_labels)
 errG.backward(retain_graph=True)
 G_running_loss += errG.item() /len(train_loader)
 optimizerG.step()
if EMA:
param_itr = cycle(netG.parameters())
 for i,p_EMA in enumerate(netG_EMA.parameters()):
p = next(param_itr)
 p_EMA.data =(1-ema_decay_rate)*p_EMA.data + ema_decay_rate*p.data
p_EMA.requires_grad = False
D_loss_list.append(D_running_loss)
 G_loss_list.append(G_running_loss)
 if epoch % output_freq == 0:
print('[{:d}/{:d}] D_loss = {:.3f}, G_loss = {:.3f}, elapsed_time = {:.1f} min'.format(epoch,epochs,D_running_loss,G_running_loss,elapsed_time(start_time)/60))
if epoch in show_epoch_list:
print('epoch = {}'.format(epoch))
if not EMA:
show_generate_imgs(netG,fixed_noise,fixed_aux_labels_ohe)
 elif EMA:
show_generate_imgs(netG_EMA,fixed_noise,fixed_aux_labels_ohe)
 if epoch % 100 == 0:
if not EMA:
torch.save(netG.state_dict() , f'generator_epoch{epoch}.pth')
 elif EMA:
torch.save(netG_EMA.state_dict() , f'generator_epoch{epoch}.pth')
 if not EMA:
torch.save(netG.state_dict() , 'generator.pth')
 elif EMA:
torch.save(netG_EMA.state_dict() , 'generator.pth')
 torch.save(netD.state_dict() , 'discriminator.pth')
 res = {'netG':netG,
'netD':netD,
'nz':nz,
'fixed_noise':fixed_noise,
'fixed_aux_labels_ohe':fixed_aux_labels_ohe,
'D_loss_list':D_loss_list,
'G_loss_list':G_loss_list,
}
if EMA:
res['netG_EMA'] = netG_EMA
return res
def make_polymodel(X, y):
with pm.Model() as model:
alpha = pm.Normal('alpha', mu=0, sd=3)
 beta1 = pm.Cauchy('beta', alpha=0, beta=.07, shape=X.shape[1])
 beta2 = pm.Cauchy('beta^2', alpha=0, beta=.07, shape=X.shape[1])
 beta3 = pm.Cauchy('beta^3', alpha=0, beta=.07, shape=X.shape[1])
 mu1 = pm.math.dot(X, beta1)
 mu2 = pm.math.dot(np.power(X,2), beta2)
 mu3 = pm.math.dot(np.power(X,3), beta3)
 p = invlogit(alpha + mu1 + mu2 + mu3)
 y_obs = pm.Bernoulli('y_obs', p=p, observed=y)
 return(model)
<load_from_csv>
%%time
show_epoch_list = np.arange(0,500+10,10)
 res = run(lr_G=3e-4,lr_D=3e-4, beta1=0.0, beta2=0.999, nz=120, epochs=500,
n_ite_D=1, ema_decay_rate=None, show_epoch_list=show_epoch_list, output_freq=10)
<create_dataframe><EOS> 
def submission_generate_images(res, truncated=None):
im_batch_size=50
n_images=10000
if not EMA:
netG = res['netG']
elif EMA:
netG = res['netG_EMA']
nz = res['nz']
if not os.path.exists('.. /output_images'):
os.mkdir('.. /output_images')
 for i_batch in range(0, n_images, im_batch_size):
if truncated is not None:
flag = True
while flag:
z = np.random.randn(100*im_batch_size*nz)
 z = z[np.where(abs(z)<truncated)]
if len(z)>=im_batch_size*nz:
flag=False
gen_z = torch.from_numpy(z[:im_batch_size*nz] ).view(im_batch_size,nz,1,1)
 gen_z = gen_z.float().to(device)
 else:
gen_z = torch.randn(im_batch_size, nz, 1, 1, device=device)
 aux_labels = np.random.randint(0,n_classes, im_batch_size)
 aux_labels_ohe = np.eye(n_classes)[aux_labels]
aux_labels_ohe = torch.from_numpy(aux_labels_ohe[:,:,np.newaxis,np.newaxis])
 aux_labels_ohe = aux_labels_ohe.float().to(device)
 gen_images = netG(gen_z,aux_labels_ohe)
 gen_images = gen_images.to("cpu" ).clone().detach()
gen_images = gen_images*0.5 + 0.5
for i_image in range(gen_images.size(0)) :
save_image(gen_images[i_image, :, :, :],
os.path.join(f'.. /output_images', f'image_{i_batch+i_image:05d}.png'))
shutil.make_archive(f'images', 'zip', f'.. /output_images' )