Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<define_variables>
ComputeLB = False
DogsOnly = False
ROOT = '.. /input/generative-dog-images/'
if not ComputeLB: ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 namesIn.append(breed)
 idxIn += 1
else:
x = np.random.choice(np.arange(20000),10000)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]; h = img.size[1];
if(k%2==0)|(k%3==0):
w2 = 100; h2 = int(h/(w/100))
a = 18; b = 0
else:
a=0; b=0
if w<h:
w2 = 64; h2 = int(( 64/w)*h)
 b =(h2-64)//2
else:
h2 = 64; w2 = int(( 64/h)*w)
 a =(w2-64)//2
img = img.resize(( w2,h2), Image.ANTIALIAS)
 img = img.crop(( 0+a, 0+b, 64+a, 64+b))
imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 idxIn += 1
x = np.random.randint(0,idxIn,25)
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = Image.fromarray(imagesIn[x[k*5+j],:,:,:].astype('uint8'))
plt.axis('off')
 if not DogsOnly: plt.title(namesIn[x[k*5+j]],fontsize=11)
 else: plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)
 plt.imshow(img)
 plt.show()
start = time()<set_options>
from keras.models import Model
from keras.layers import Input, Dense, Conv2D, MaxPooling2D, UpSampling2D, Reshape, BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
from keras.optimizers import SGD, Adam
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<load_pretrained>
direct_input = Input(( 10000,))
x = Dense(2048, activation='elu' )(direct_input)
 x = Reshape(( 8,8,32))(x)
 x = Conv2D(128,(3, 3), activation='elu', padding='same' )(x)
 x = UpSampling2D(( 2, 2))(x)
 x = Conv2D(64,(3, 3), activation='elu', padding='same' )(x)
 x = UpSampling2D(( 2, 2))(x)
 x = Conv2D(32,(3, 3), activation='elu', padding='same' )(x)
 x = UpSampling2D(( 2, 2))(x)
 decoded = Conv2D(3,(3, 3), activation='sigmoid', padding='same' )(x)
 decoder = Model(direct_input, decoded)
 decoder.compile(optimizer=Adam(lr=0.003), loss='binary_crossentropy')
 decoder.summary()
class DataGenerator(Dataset):
def __init__(self, directory, transform=None, n_samples=np.inf):
self.directory = directory
self.transform = transform
self.n_samples = n_samples
self.samples = self._load_subfolders_images(directory)
 if len(self.samples)== 0:
raise RuntimeError("Found 0 files in subfolders of: {}".format(directory))
def _load_subfolders_images(self, root):
IMG_EXTENSIONS =(
 '.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
 def is_valid_file(x):
return torchvision.datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 required_transforms = torchvision.transforms.Compose([
torchvision.transforms.Resize(64),
torchvision.transforms.CenterCrop(64),
])
 imgs = []
paths = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames)[:min(self.n_samples, 999999999999999)]:
path = os.path.join(root, fname)
 paths.append(path)
 for path in paths:
if is_valid_file(path):
img = dset.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(
 dirname for dirname in os.listdir('.. /input/annotation/Annotation/')if
dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join('.. /input/annotation/Annotation/',
annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
bbox =(xmin, ymin, xmin+w, ymin+w)
 object_img = required_transforms(img.crop(bbox))
imgs.append(object_img)
 return imgs
def __getitem__(self, index):
sample = self.samples[index]
if self.transform is not None:
sample = self.transform(sample)
 return np.asarray(sample)
 def __len__(self):
return len(self.samples)
<categorify>
idx = np.random.randint(0,idxIn,10000)
 train_y = imagesIn[idx,:,:,:]/255.
 train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1
%%time
database = '.. /input/all-dogs/all-dogs/'
transform = transforms.Compose([transforms.RandomHorizontalFlip(p=0.3),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 train_data = DataGenerator(database, transform=transform,n_samples=25000)
 train_loader = torch.utils.data.DataLoader(train_data, shuffle=True,batch_size=batch_size, num_workers = 4)
<normalization>
lr = 0.005
for k in range(50):
annealer = LearningRateScheduler(lambda x: lr)
 h = decoder.fit(train_X, train_y, epochs = 20, batch_size=256, callbacks=[annealer], verbose=0)
 if k%5==4: print('Epoch',(k+1)*10,'/500 - loss =',h.history['loss'][-1])
 if h.history['loss'][-1]<0.54: lr = 0.001
class PixelwiseNorm(nn.Module):
def __init__(self):
super(PixelwiseNorm, self ).__init__()
def forward(self, x, alpha=1e-8):

 y = x.pow(2.).mean(dim=1, keepdim=True ).add(alpha ).sqrt()
y = x / y
return y<categorify>
del train_X, train_y, imagesIn
class MinibatchStdDev(th.nn.Module):

 def __init__(self):

 super(MinibatchStdDev, self ).__init__()
def forward(self, x, alpha=1e-8):

 batch_size, _, height, width = x.shape
y = x - x.mean(dim=0, keepdim=True)
 y = th.sqrt(y.pow(2.).mean(dim=0, keepdim=False)+ alpha)
 y = y.mean().view(1, 1, 1, 1)
 y = y.repeat(batch_size,1, height, width)
 y = th.cat([x, y], 1)
 return y 
<define_search_model>
print('Generate Random Dogs')
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
xx[np.random.randint(10000)] = 0.75
xx = xx/(np.sqrt(xx.dot(xx.T)))
 plt.subplot(1,5,j+1)
 img = decoder.predict(xx.reshape(( -1,10000)))
 img = Image.fromarray(( 255*img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()
class Generator(nn.Module):
def __init__(self, nz, nfeats, nchannels):
super(Generator, self ).__init__()
self.conv1 = spectral_norm(nn.ConvTranspose2d(nz, nfeats * 8, 4, 1, 0, bias=False))
self.conv2 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 8, 4, 2, 1, bias=False))
self.conv3 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 4, 4, 2, 1, bias=False))
self.conv4 = spectral_norm(nn.ConvTranspose2d(nfeats * 4, nfeats * 2, 4, 2, 1, bias=False))
self.conv5 = spectral_norm(nn.ConvTranspose2d(nfeats * 2, nfeats, 4, 2, 1, bias=False))
self.conv6 = spectral_norm(nn.ConvTranspose2d(nfeats, nchannels, 3, 1, 1, bias=False))
self.pixnorm = PixelwiseNorm()
def forward(self, x):
x = F.leaky_relu(self.conv1(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv2(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv3(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv4(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv5(x))
x = self.pixnorm(x)
 x = torch.tanh(self.conv6(x))
return x
class Discriminator(nn.Module):
def __init__(self, nchannels, nfeats):
super(Discriminator, self ).__init__()
self.conv1 = nn.Conv2d(nchannels, nfeats, 4, 2, 1, bias=False)
 self.conv2 = spectral_norm(nn.Conv2d(nfeats, nfeats * 2, 4, 2, 1, bias=False))
self.bn2 = nn.BatchNorm2d(nfeats * 2)
 self.conv3 = spectral_norm(nn.Conv2d(nfeats * 2, nfeats * 4, 4, 2, 1, bias=False))
self.bn3 = nn.BatchNorm2d(nfeats * 4)
 self.conv4 = spectral_norm(nn.Conv2d(nfeats * 4, nfeats * 8, 4, 2, 1, bias=False))
self.bn4 = nn.MaxPool2d(2)
 self.batch_discriminator = MinibatchStdDev()
self.pixnorm = PixelwiseNorm()
self.conv5 = spectral_norm(nn.Conv2d(nfeats * 8 +1, 1, 2, 1, 0, bias=False))
def forward(self, x):
x = F.leaky_relu(self.conv1(x), 0.2)
 x = F.leaky_relu(self.bn2(self.conv2(x)) , 0.2)
 x = F.leaky_relu(self.bn3(self.conv3(x)) , 0.2)
 x = F.leaky_relu(self.bn4(self.conv4(x)) , 0.2)
 x = self.batch_discriminator(x)
 x = torch.sigmoid(self.conv5(x))
return x.view(-1, 1)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 lr = 0.00002
lr_d = 0.00005
beta1 = 0.25
epochs = 1000
nz = 10
netG = Generator(nz, 32, 3 ).to(device)
 netD = Discriminator(3, 48 ).to(device)
 criterion = nn.BCELoss()
optimizerD = optim.Adam(netD.parameters() , lr=lr, betas=(beta1, 0.999))
optimizerG = optim.Adam(netG.parameters() , lr=lr_d, betas=(beta1, 0.999))
lr_schedulerG = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerG,
T_0=30, eta_min=0.00005)
 lr_schedulerD = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerD,
T_0=30, eta_min=0.00005)
 fixed_noise = torch.randn(25, nz, 1, 1, device=device)
 real_label = 0.9
fake_label = 0.
 batch_size = train_loader.batch_size
step = 0
min_error = 1e6
for epoch in range(epochs):
for ii,(real_images)in tqdm(enumerate(train_loader), total=len(train_loader)) :
end = time()
if(end -start)> 25000 :
break
netD.zero_grad()
real_images = real_images.to(device)
 batch_size = real_images.size(0)
 labels = torch.full(( batch_size, 1), real_label, device=device)+ np.random.uniform(-0.1, 0.1)
 output = netD(real_images)
 errD_real = criterion(output, labels)
 errD_real.backward()
D_x = output.mean().item()
noise = torch.randn(batch_size, nz, 1, 1, device=device)
 fake = netG(noise)
 labels.fill_(fake_label)+ np.random.uniform(0, 0.2)
 output = netD(fake.detach())
 errD_fake = criterion(output, labels)
 errD_fake.backward()
D_G_z1 = output.mean().item()
errD = errD_real + errD_fake
optimizerD.step()
netG.zero_grad()
labels.fill_(real_label)
 output = netD(fake)
 errG = criterion(output, labels)
 errG.backward()
D_G_z2 = output.mean().item()
optimizerG.step()
if step % 500 == 0:
print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)) : %.4f / %.4f'
%(epoch + 1, epochs, ii, len(train_loader),
errD.item() , errG.item() , D_x, D_G_z1, D_G_z2))
valid_image = netG(fixed_noise)
 step += 1
lr_schedulerG.step(epoch)
 lr_schedulerD.step(epoch)
 if epoch % 20 == 0:
show_generated_img() 
<compute_test_metric>
print('Recall from Memory Dogs')
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 img = decoder.predict(xx.reshape(( -1,10000)))
 img = Image.fromarray(( 255*img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()
print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)) : %.4f / %.4f'
%(epoch + 1, epochs, ii, len(train_loader),
errD.item() , errG.item() , D_x, D_G_z1, D_G_z2))<categorify>
for k in range(6):
if(k==0)|(k==2)|(k==4):
print('Walk in Latent Space')
 a = np.random.randint(10000)
 b = np.random.randint(10000)
 plt.figure(figsize=(15,3))
for j in range(5):
xx = np.zeros(( 10000))
theta =(( k%2)*5+j)/9
xx[a] = theta; xx[b] = 1-theta
xx = xx/(np.sqrt(xx.dot(xx.T)))
 plt.subplot(1,5,j+1)
 img = decoder.predict(xx.reshape(( -1,10000)))
 img = Image.fromarray(( 255*img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values
if not os.path.exists('.. /output_images'):
os.mkdir('.. /output_images')
 im_batch_size = 100
n_images=10000
for i_batch in range(0, n_images, im_batch_size):
z = truncated_normal(( im_batch_size, nz, 1, 1), threshold=1)
 gen_z = torch.from_numpy(z ).float().to(device)
 gen_images = netG(gen_z)
 images = gen_images.to("cpu" ).clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
 for i_image in range(gen_images.size(0)) :
save_image(( gen_images[i_image, :, :, :] +1.0)/2.0, os.path.join('.. /output_images', f'image_{i_batch+i_image:05d}.png'))
shutil.make_archive('images', 'zip', '.. /output_images' )<init_hyperparams>
z = zipfile.PyZipFile('images.zip', mode='w')
 for k in range(10000):
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 0.99
xx[np.random.randint(10000)] = 0.01
img = decoder.predict(xx.reshape(( -1,10000)))
 img = Image.fromarray(( 255*img ).astype('uint8' ).reshape(( 64,64,3)))
 f = str(k)+'.png'
img.save(f,'PNG'); z.write(f); os.remove(f)
 z.close()
batch_size = 16
nz = 100
lr = 0.0003
lr_d = 0.0001
beta1 = 0.5
epochs = 1000<init_hyperparams>
class KernelEvalException(Exception):
pass
model_params = {
'Inception': {
'name': 'Inception',
'imsize': 64,
'output_layer': 'Pretrained_Net/pool_3:0',
'input_layer': 'Pretrained_Net/ExpandDims:0',
'output_shape': 2048,
'cosine_distance_eps': 0.1
}
}
def create_model_graph(pth):

 with tf.gfile.FastGFile(pth, 'rb')as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
 _ = tf.import_graph_def(graph_def, name='Pretrained_Net')
 def _get_model_layer(sess, model_name):
layername = model_params[model_name]['output_layer']
layer = sess.graph.get_tensor_by_name(layername)
 ops = layer.graph.get_operations()
for op_idx, op in enumerate(ops):
for o in op.outputs:
shape = o.get_shape()
if shape._dims != []:
shape = [s.value for s in shape]
new_shape = []
for j, s in enumerate(shape):
if s == 1 and j == 0:
new_shape.append(None)
 else:
new_shape.append(s)
 o.__dict__['_shape_val'] = tf.TensorShape(new_shape)
 return layer
def get_activations(images, sess, model_name, batch_size=50, verbose=False):

 inception_layer = _get_model_layer(sess, model_name)
 n_images = images.shape[0]
if batch_size > n_images:
print("warning: batch size is bigger than the data size.setting batch size to data size")
 batch_size = n_images
n_batches = n_images//batch_size + 1
pred_arr = np.empty(( n_images,model_params[model_name]['output_shape']))
for i in tqdm(range(n_batches)) :
if verbose:
print("\rPropagating batch %d/%d" %(i+1, n_batches), end="", flush=True)
 start = i*batch_size
if start+batch_size < n_images:
end = start+batch_size
else:
end = n_images
batch = images[start:end]
pred = sess.run(inception_layer, {model_params[model_name]['input_layer']: batch})
 pred_arr[start:end] = pred.reshape(-1,model_params[model_name]['output_shape'])
 if verbose:
print(" done")
 return pred_arr
def normalize_rows(x: np.ndarray):

 return np.nan_to_num(x/np.linalg.norm(x, ord=2, axis=1, keepdims=True))
def cosine_distance(features1, features2):
features1_nozero = features1[np.sum(features1, axis=1)!= 0]
features2_nozero = features2[np.sum(features2, axis=1)!= 0]
norm_f1 = normalize_rows(features1_nozero)
 norm_f2 = normalize_rows(features2_nozero)
 d = 1.0-np.abs(np.matmul(norm_f1, norm_f2.T))
print('d.shape=',d.shape)
 print('np.min(d, axis=1 ).shape=',np.min(d, axis=1 ).shape)
 mean_min_d = np.mean(np.min(d, axis=1))
print('distance=',mean_min_d)
 return mean_min_d
def distance_thresholding(d, eps):
if d < eps:
return d
else:
return 1
def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):

 mu1 = np.atleast_1d(mu1)
 mu2 = np.atleast_1d(mu2)
 sigma1 = np.atleast_2d(sigma1)
 sigma2 = np.atleast_2d(sigma2)
 assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"
diff = mu1 - mu2
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
 if not np.isfinite(covmean ).all() :
msg = "fid calculation produces singular product; adding %s to diagonal of cov estimates" % eps
warnings.warn(msg)
 offset = np.eye(sigma1.shape[0])* eps
covmean = linalg.sqrtm(( sigma1 + offset ).dot(sigma2 + offset))
if np.iscomplexobj(covmean):
if not np.allclose(np.diagonal(covmean ).imag, 0, atol=1e-3):
m = np.max(np.abs(covmean.imag))
raise ValueError("Imaginary component {}".format(m))
covmean = covmean.real
print('covmean.shape=',covmean.shape)
 tr_covmean = np.trace(covmean)
 return diff.dot(diff)+ np.trace(sigma1)+ np.trace(sigma2)- 2 * tr_covmean
def calculate_activation_statistics(images, sess, model_name, batch_size=50, verbose=False):

 act = get_activations(images, sess, model_name, batch_size, verbose)
 mu = np.mean(act, axis=0)
 sigma = np.cov(act, rowvar=False)
 return mu, sigma, act
def _handle_path_memorization(path, sess, model_name, is_checksize, is_check_png):
path = pathlib.Path(path)
 files = list(path.glob('*.jpg')) + list(path.glob('*.png'))
imsize = model_params[model_name]['imsize']
x = np.array([np.array(img_read_checks(fn, imsize, is_checksize, imsize, is_check_png)) for fn in files])
 m, s, features = calculate_activation_statistics(x, sess, model_name)
 del x
return m, s, features
def img_read_checks(filename, resize_to, is_checksize=False, check_imsize = 64, is_check_png = False):
im = Image.open(str(filename))
if is_checksize and im.size !=(check_imsize,check_imsize):
raise KernelEvalException('The images are not of size '+str(check_imsize))
if is_check_png and im.format != 'PNG':
raise KernelEvalException('Only PNG images should be submitted.')
 if resize_to is None:
return im
else:
return im.resize(( resize_to,resize_to),Image.ANTIALIAS)
 def calculate_kid_given_paths(paths, model_name, model_path, feature_path=None, mm=[], ss=[], ff=[]):

 tf.reset_default_graph()
create_model_graph(str(model_path))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
 m1, s1, features1 = _handle_path_memorization(paths[0], sess, model_name, is_checksize = True, is_check_png = True)
 if len(mm)!= 0:
m2 = mm
s2 = ss
features2 = ff
elif feature_path is None:
m2, s2, features2 = _handle_path_memorization(paths[1], sess, model_name, is_checksize = False, is_check_png = False)
 else:
with np.load(feature_path)as f:
m2, s2, features2 = f['m'], f['s'], f['features']
print('m1,m2 shape=',(m1.shape,m2.shape),'s1,s2=',(s1.shape,s2.shape))
print('starting calculating FID')
 fid_value = calculate_frechet_distance(m1, s1, m2, s2)
 print('done with FID, starting distance calculation')
 distance = cosine_distance(features1, features2)
 return fid_value, distance, m2, s2, features2
class AdaBound(Optimizer):

 def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), final_lr=0.1, gamma=1e-3,
eps=1e-8, weight_decay=0, amsbound=False):
if not 0.0 <= lr:
raise ValueError("Invalid learning rate: {}".format(lr))
if not 0.0 <= eps:
raise ValueError("Invalid epsilon value: {}".format(eps))
if not 0.0 <= betas[0] < 1.0:
raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
if not 0.0 <= betas[1] < 1.0:
raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
if not 0.0 <= final_lr:
raise ValueError("Invalid final learning rate: {}".format(final_lr))
if not 0.0 <= gamma < 1.0:
raise ValueError("Invalid gamma parameter: {}".format(gamma))
defaults = dict(lr=lr, betas=betas, final_lr=final_lr, gamma=gamma, eps=eps,
weight_decay=weight_decay, amsbound=amsbound)
 super(AdaBound, self ).__init__(params, defaults)
 self.base_lrs = list(map(lambda group: group['lr'], self.param_groups))
def __setstate__(self, state):
super(AdaBound, self ).__setstate__(state)
 for group in self.param_groups:
group.setdefault('amsbound', False)
 def step(self, closure=None):

 loss = None
if closure is not None:
loss = closure()
for group, base_lr in zip(self.param_groups, self.base_lrs):
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data
if grad.is_sparse:
raise RuntimeError(
 'Adam does not support sparse gradients, please consider SparseAdam instead')
 amsbound = group['amsbound']
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p.data)
 state['exp_avg_sq'] = torch.zeros_like(p.data)
 if amsbound:
state['max_exp_avg_sq'] = torch.zeros_like(p.data)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
if amsbound:
max_exp_avg_sq = state['max_exp_avg_sq']
beta1, beta2 = group['betas']
state['step'] += 1
if group['weight_decay'] != 0:
grad = grad.add(group['weight_decay'], p.data)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 if amsbound:
torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)
 denom = max_exp_avg_sq.sqrt().add_(group['eps'])
 else:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 bias_correction1 = 1 - beta1 ** state['step']
bias_correction2 = 1 - beta2 ** state['step']
step_size = group['lr'] * math.sqrt(bias_correction2)/ bias_correction1
final_lr = group['final_lr'] * group['lr'] / base_lr
lower_bound = final_lr *(1 - 1 /(group['gamma'] * state['step'] + 1))
upper_bound = final_lr *(1 + 1 /(group['gamma'] * state['step']))
step_size = torch.full_like(denom, step_size)
 step_size.div_(denom ).clamp_(lower_bound, upper_bound ).mul_(exp_avg)
 p.data.add_(-step_size)
 return loss
class AdaBoundW(Optimizer):

 def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), final_lr=0.1, gamma=1e-3,
eps=1e-8, weight_decay=0, amsbound=False):
if not 0.0 <= lr:
raise ValueError("Invalid learning rate: {}".format(lr))
if not 0.0 <= eps:
raise ValueError("Invalid epsilon value: {}".format(eps))
if not 0.0 <= betas[0] < 1.0:
raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
if not 0.0 <= betas[1] < 1.0:
raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
if not 0.0 <= final_lr:
raise ValueError("Invalid final learning rate: {}".format(final_lr))
if not 0.0 <= gamma < 1.0:
raise ValueError("Invalid gamma parameter: {}".format(gamma))
defaults = dict(lr=lr, betas=betas, final_lr=final_lr, gamma=gamma, eps=eps,
weight_decay=weight_decay, amsbound=amsbound)
 super(AdaBoundW, self ).__init__(params, defaults)
 self.base_lrs = list(map(lambda group: group['lr'], self.param_groups))
def __setstate__(self, state):
super(AdaBoundW, self ).__setstate__(state)
 for group in self.param_groups:
group.setdefault('amsbound', False)
 def step(self, closure=None):

 loss = None
if closure is not None:
loss = closure()
for group, base_lr in zip(self.param_groups, self.base_lrs):
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data
if grad.is_sparse:
raise RuntimeError(
 'Adam does not support sparse gradients, please consider SparseAdam instead')
 amsbound = group['amsbound']
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p.data)
 state['exp_avg_sq'] = torch.zeros_like(p.data)
 if amsbound:
state['max_exp_avg_sq'] = torch.zeros_like(p.data)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
if amsbound:
max_exp_avg_sq = state['max_exp_avg_sq']
beta1, beta2 = group['betas']
state['step'] += 1
exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 if amsbound:
torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)
 denom = max_exp_avg_sq.sqrt().add_(group['eps'])
 else:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 bias_correction1 = 1 - beta1 ** state['step']
bias_correction2 = 1 - beta2 ** state['step']
step_size = group['lr'] * math.sqrt(bias_correction2)/ bias_correction1
final_lr = group['final_lr'] * group['lr'] / base_lr
lower_bound = final_lr *(1 - 1 /(group['gamma'] * state['step'] + 1))
upper_bound = final_lr *(1 + 1 /(group['gamma'] * state['step']))
step_size = torch.full_like(denom, step_size)
 step_size.div_(denom ).clamp_(lower_bound, upper_bound ).mul_(exp_avg)
 if group['weight_decay'] != 0:
decayed_weights = torch.mul(p.data, group['weight_decay'])
 p.data.add_(-step_size)
 p.data.sub_(decayed_weights)
 else:
p.data.add_(-step_size)
 return loss 
<import_modules>
if ComputeLB:
with zipfile.ZipFile(".. /working/images.zip","r")as z:
z.extractall(".. /tmp/images2/")
 m2 = []; s2 =[]; f2 = []
user_images_unzipped_path = '.. /tmp/images2/'
images_path = [user_images_unzipped_path,'.. /input/generative-dog-images/all-dogs/all-dogs/']
public_path = '.. /input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'
fid_epsilon = 10e-15
fid_value_public, distance_public, m2, s2, f2 = calculate_kid_given_paths(images_path, 'Inception', public_path, mm=m2, ss=s2, ff=f2)
 distance_public = distance_thresholding(distance_public, model_params['Inception']['cosine_distance_eps'])
 print("FID_public: ", fid_value_public, "distance_public: ", distance_public, "multiplied_public: ",
fid_value_public /(distance_public + fid_epsilon))
! rm -r.. /tmp