Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<train_model>
import os
import numpy as np
import pandas as pd
import random
from tqdm import tqdm
import shutil
clf_svc.fit(train_data.clip(0,100000000)[0:8000],train_label[0:8000] )<train_model>
class KernelEvalException(Exception):
pass
model_params = {
'Inception': {
'name': 'Inception',
'imsize': 64,
'output_layer': 'Pretrained_Net/pool_3:0',
'input_layer': 'Pretrained_Net/ExpandDims:0',
'output_shape': 2048,
'cosine_distance_eps': 0.1
}
}
def create_model_graph(pth):

 with tf.gfile.FastGFile(pth, 'rb')as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
 _ = tf.import_graph_def(graph_def, name='Pretrained_Net')
 def _get_model_layer(sess, model_name):
layername = model_params[model_name]['output_layer']
layer = sess.graph.get_tensor_by_name(layername)
 ops = layer.graph.get_operations()
for op_idx, op in enumerate(ops):
for o in op.outputs:
shape = o.get_shape()
if shape._dims != []:
shape = [s.value for s in shape]
new_shape = []
for j, s in enumerate(shape):
if s == 1 and j == 0:
new_shape.append(None)
 else:
new_shape.append(s)
 o.__dict__['_shape_val'] = tf.TensorShape(new_shape)
 return layer
def get_activations(images, sess, model_name, batch_size=50, verbose=False):

 inception_layer = _get_model_layer(sess, model_name)
 n_images = images.shape[0]
if batch_size > n_images:
print("warning: batch size is bigger than the data size.setting batch size to data size")
 batch_size = n_images
n_batches = n_images//batch_size + 1
pred_arr = np.empty(( n_images,model_params[model_name]['output_shape']))
for i in tqdm(range(n_batches)) :
if verbose:
print("\rPropagating batch %d/%d" %(i+1, n_batches), end="", flush=True)
 start = i*batch_size
if start+batch_size < n_images:
end = start+batch_size
else:
end = n_images
batch = images[start:end]
pred = sess.run(inception_layer, {model_params[model_name]['input_layer']: batch})
 pred_arr[start:end] = pred.reshape(-1,model_params[model_name]['output_shape'])
 if verbose:
print(" done")
 return pred_arr
def normalize_rows(x: np.ndarray):

 return np.nan_to_num(x/np.linalg.norm(x, ord=2, axis=1, keepdims=True))
def cosine_distance(features1, features2):
features1_nozero = features1[np.sum(features1, axis=1)!= 0]
features2_nozero = features2[np.sum(features2, axis=1)!= 0]
norm_f1 = normalize_rows(features1_nozero)
 norm_f2 = normalize_rows(features2_nozero)
 d = 1.0-np.abs(np.matmul(norm_f1, norm_f2.T))
print('d.shape=',d.shape)
 print('np.min(d, axis=1 ).shape=',np.min(d, axis=1 ).shape)
 mean_min_d = np.mean(np.min(d, axis=1))
print('distance=',mean_min_d)
 return mean_min_d
def distance_thresholding(d, eps):
if d < eps:
return d
else:
return 1
def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):

 mu1 = np.atleast_1d(mu1)
 mu2 = np.atleast_1d(mu2)
 sigma1 = np.atleast_2d(sigma1)
 sigma2 = np.atleast_2d(sigma2)
 assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"
diff = mu1 - mu2
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
 if not np.isfinite(covmean ).all() :
msg = "fid calculation produces singular product; adding %s to diagonal of cov estimates" % eps
warnings.warn(msg)
 offset = np.eye(sigma1.shape[0])* eps
covmean = linalg.sqrtm(( sigma1 + offset ).dot(sigma2 + offset))
if np.iscomplexobj(covmean):
if not np.allclose(np.diagonal(covmean ).imag, 0, atol=1e-3):
m = np.max(np.abs(covmean.imag))
raise ValueError("Imaginary component {}".format(m))
covmean = covmean.real
print('covmean.shape=',covmean.shape)
 tr_covmean = np.trace(covmean)
 return diff.dot(diff)+ np.trace(sigma1)+ np.trace(sigma2)- 2 * tr_covmean
def calculate_activation_statistics(images, sess, model_name, batch_size=50, verbose=False):

 act = get_activations(images, sess, model_name, batch_size, verbose)
 mu = np.mean(act, axis=0)
 sigma = np.cov(act, rowvar=False)
 return mu, sigma, act
def _handle_path_memorization(path, sess, model_name, is_checksize, is_check_png):
path = pathlib.Path(path)
 files = list(path.glob('*.jpg')) + list(path.glob('*.png'))
imsize = model_params[model_name]['imsize']
x = np.array([np.array(img_read_checks(fn, imsize, is_checksize, imsize, is_check_png)) for fn in files])
 m, s, features = calculate_activation_statistics(x, sess, model_name)
 del x
return m, s, features
def img_read_checks(filename, resize_to, is_checksize=False, check_imsize = 64, is_check_png = False):
im = Image.open(str(filename))
if is_checksize and im.size !=(check_imsize,check_imsize):
raise KernelEvalException('The images are not of size '+str(check_imsize))
if is_check_png and im.format != 'PNG':
raise KernelEvalException('Only PNG images should be submitted.')
 if resize_to is None:
return im
else:
return im.resize(( resize_to,resize_to),Image.ANTIALIAS)
 def calculate_kid_given_paths(paths, model_name, model_path, feature_path=None):

 tf.reset_default_graph()
create_model_graph(str(model_path))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
 m1, s1, features1 = _handle_path_memorization(paths[0], sess, model_name, is_checksize = True, is_check_png = True)
 if feature_path is None:
m2, s2, features2 = _handle_path_memorization(paths[1], sess, model_name, is_checksize = False, is_check_png = False)
 else:
with np.load(feature_path)as f:
m2, s2, features2 = f['m'], f['s'], f['features']
print('m1,m2 shape=',(m1.shape,m2.shape),'s1,s2=',(s1.shape,s2.shape))
print('starting calculating FID')
 fid_value = calculate_frechet_distance(m1, s1, m2, s2)
 print('done with FID, starting distance calculation')
 distance = cosine_distance(features1, features2)
 return fid_value, distance
clf_svr.fit(train_data.clip(0,100000000)[0:8000],train_label[0:8000] )<predict_on_test>
dogs = os.listdir('.. /input/generative-dog-images/all-dogs/all-dogs/')
 print("Total images = ", len(dogs))
sub = random.sample(dogs, 10000)
 print("Sample images = ", len(sub))
f1_score(clf_svc.predict(train_data.clip(0,100000000)) , train_label,average='macro' )<predict_on_test>
path = '/kaggle/working/images/'
def resize() :
for item in tqdm(os.listdir(path)) :
im = Image.open(path+item)
 imResize = im.resize(( 64,64), Image.ANTIALIAS)
 imResize.save(path+item, 'PNG', quality=100)
 resize()
f1_score(np.round(clf_svr.predict(train_data.clip(0,100000000)) ).clip(1,4), train_label,average='macro' )<predict_on_test>
user_images_unzipped_path = 'images'
images_path = [user_images_unzipped_path,'.. /input/generative-dog-images/all-dogs/all-dogs/']
public_path = '.. /input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'
fid_epsilon = 10e-15
fid_value_public, distance_public = calculate_kid_given_paths(images_path, 'Inception', public_path)
 distance_public = distance_thresholding(distance_public, model_params['Inception']['cosine_distance_eps'])
 print("FID_public: ", fid_value_public, "distance_public: ", distance_public, "multiplied_public: ", fid_value_public /(distance_public + fid_epsilon))
pre_svc = clf_svc.predict(test_data.clip(0,100000000))
pre_svr = np.array(np.round(clf_svr.predict(test_data.clip(0,100000000)) ).clip(1,4),dtype='int64' )<create_dataframe>
!rm -r images
pre_svc_df = pd.DataFrame({'Id':test_Id, 'Target':pre_svc})
 pre_svr_df = pd.DataFrame({'Id':test_Id, 'Target':pre_svr} )<save_to_csv>
ComputeLB = True
DogsOnly = False
ROOT = '.. /input/generative-dog-images/'
if not ComputeLB: ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 namesIn.append(breed)
 idxIn += 1
idx = np.arange(idxIn)
 np.random.shuffle(idx)
 imagesIn = imagesIn[idx,:,:,:]
namesIn = np.array(namesIn)[idx]
else:
IMAGES = np.sort(IMAGES)
 np.random.seed(810)
 x = np.random.choice(np.arange(20579),10000)
 np.random.seed(None)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]; h = img.size[1];
if(k%2==0)|(k%3==0):
w2 = 100; h2 = int(h/(w/100))
a = 18; b = 0
else:
a=0; b=0
if w<h:
w2 = 64; h2 = int(( 64/w)*h)
 b =(h2-64)//2
else:
h2 = 64; w2 = int(( 64/h)*w)
 a =(w2-64)//2
img = img.resize(( w2,h2), Image.ANTIALIAS)
 img = img.crop(( 0+a, 0+b, 64+a, 64+b))
imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 idxIn += 1
x = np.random.randint(0,idxIn,25)
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = Image.fromarray(imagesIn[x[k*5+j],:,:,:].astype('uint8'))
plt.axis('off')
 if not DogsOnly: plt.title(namesIn[x[k*5+j]],fontsize=11)
 else: plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)
 plt.imshow(img)
 plt.show()
pre_svc_df.to_csv('submission_svc.csv',index=False)
 pre_svr_df.to_csv('submission_svr.csv',index=False )<set_options>
from keras.models import Model
from keras.layers import Input, Dense, Conv2D, Reshape, Flatten, concatenate
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
from keras.optimizers import SGD, Adam
sns.set()
%matplotlib inline
labels = {'v2a1':' Monthly rent payment','hacdor':' =1 Overcrowding by bedrooms','rooms':' number of all rooms in the house','hacapo':' =1 Overcrowding by rooms','v14a':' =1 has toilet in the household','refrig':' =1 if the household has refrigerator','v18q':' owns a tablet','v18q1':' number of tablets household owns','r4h1':' Males younger than 12 years of age','r4h2':' Males 12 years of age and older','r4h3':' Total males in the household','r4m1':' Females younger than 12 years of age','r4m2':' Females 12 years of age and older','r4m3':' Total females in the household','r4t1':' persons younger than 12 years of age','r4t2':' persons 12 years of age and older','r4t3':' Total persons in the household','tamhog':' size of the household','tamviv':' TamViv','escolari':' years of schooling','rez_esc':' Years behind in school','hhsize':' household size','paredblolad':' =1 if predominant material on the outside wall is block or brick','paredzocalo':' "=1 if predominant material on the outside wall is socket(wood zinc or absbesto"','paredpreb':' =1 if predominant material on the outside wall is prefabricated or cement','pareddes':' =1 if predominant material on the outside wall is waste material','paredmad':' =1 if predominant material on the outside wall is wood','paredzinc':' =1 if predominant material on the outside wall is zink','paredfibras':' =1 if predominant material on the outside wall is natural fibers','paredother':' =1 if predominant material on the outside wall is other','pisomoscer':' "=1 if predominant material on the floor is mosaic ceramic terrazo"','pisocemento':' =1 if predominant material on the floor is cement','pisoother':' =1 if predominant material on the floor is other','pisonatur':' =1 if predominant material on the floor is natural material','pisonotiene':' =1 if no floor at the household','pisomadera':' =1 if predominant material on the floor is wood','techozinc':' =1 if predominant material on the roof is metal foil or zink','techoentrepiso':' "=1 if predominant material on the roof is fiber cement mezzanine "','techocane':' =1 if predominant material on the roof is natural fibers','techootro':' =1 if predominant material on the roof is other','cielorazo':' =1 if the house has ceiling','abastaguadentro':' =1 if water provision inside the dwelling','abastaguafuera':' =1 if water provision outside the dwelling','abastaguano':' =1 if no water provision','public':' "=1 electricity from CNFL ICE ESPH/JASEC"','planpri':' =1 electricity from private plant','noelec':' =1 no electricity in the dwelling','coopele':' =1 electricity from cooperative','sanitario1':' =1 no toilet in the dwelling','sanitario2':' =1 toilet connected to sewer or cesspool','sanitario3':' =1 toilet connected to septic tank','sanitario5':' =1 toilet connected to black hole or letrine','sanitario6':' =1 toilet connected to other system','energcocinar1':' =1 no main source of energy used for cooking(no kitchen)','energcocinar2':' =1 main source of energy used for cooking electricity','energcocinar3':' =1 main source of energy used for cooking gas','energcocinar4':' =1 main source of energy used for cooking wood charcoal','elimbasu1':' =1 if rubbish disposal mainly by tanker truck','elimbasu2':' =1 if rubbish disposal mainly by botan hollow or buried','elimbasu3':' =1 if rubbish disposal mainly by burning','elimbasu4':' =1 if rubbish disposal mainly by throwing in an unoccupied space','elimbasu5':' "=1 if rubbish disposal mainly by throwing in river creek or sea"','elimbasu6':' =1 if rubbish disposal mainly other','epared1':' =1 if walls are bad','epared2':' =1 if walls are regular','epared3':' =1 if walls are good','etecho1':' =1 if roof are bad','etecho2':' =1 if roof are regular','etecho3':' =1 if roof are good','eviv1':' =1 if floor are bad','eviv2':' =1 if floor are regular','eviv3':' =1 if floor are good','dis':' =1 if disable person','male':' =1 if male','female':' =1 if female','estadocivil1':' =1 if less than 10 years old','estadocivil2':' =1 if free or coupled uunion','estadocivil3':' =1 if married','estadocivil4':' =1 if divorced','estadocivil5':' =1 if separated','estadocivil6':' =1 if widow/er','estadocivil7':' =1 if single','parentesco1':' =1 if household head','parentesco2':' =1 if spouse/partner','parentesco3':' =1 if son/doughter','parentesco4':' =1 if stepson/doughter','parentesco5':' =1 if son/doughter in law','parentesco6':' =1 if grandson/doughter','parentesco7':' =1 if mother/father','parentesco8':' =1 if father/mother in law','parentesco9':' =1 if brother/sister','parentesco10':' =1 if brother/sister in law','parentesco11':' =1 if other family member','parentesco12':' =1 if other non family member','idhogar':' Household level identifier','hogar_nin':' Number of children 0 to 19 in household','hogar_adul':' Number of adults in household','hogar_mayor':' 
<feature_engineering>
def define_discriminator(in_shape=(28,28,1)) :
	model = Sequential()
	
	model.add(Conv2D(128,(3,3), strides=(2,2), padding='same', input_shape=in_shape))
	model.add(LeakyReLU(alpha=0.2))
	
	model.add(Conv2D(128,(3,3), strides=(2,2), padding='same'))
	model.add(LeakyReLU(alpha=0.2))
	
	model.add(Flatten())
 	model.add(Dropout(0.4))
	model.add(Dense(1, activation='sigmoid'))
	
	opt = Adam(lr=0.0002, beta_1=0.5)
 	model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])
 	return model
df = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 df['Target0']=df['Target']-1
locations = ['lugar1','lugar2','lugar3','lugar4','lugar5','lugar6']
le = LabelEncoder()
df['idhogarnum'] = df[['idhogar']].apply(le.fit_transform)
 df_test['idhogarnum'] = df_test[['idhogar']].apply(le.fit_transform )<categorify>
def define_discriminator(in_shape=(28,28,1), n_classes=10):
	
	in_label = Input(shape=(1,))
	
	li = Embedding(n_classes, 50 )(in_label)
 	
	n_nodes = in_shape[0] * in_shape[1]
	li = Dense(n_nodes )(li)
 	
	li = Reshape(( in_shape[0], in_shape[1], 1))(li)
 	
	in_image = Input(shape=in_shape)
 	
	merge = Concatenate()([in_image, li])
 	
	fe = Conv2D(128,(3,3), strides=(2,2), padding='same' )(merge)
 	fe = LeakyReLU(alpha=0.2 )(fe)
 	
	fe = Conv2D(128,(3,3), strides=(2,2), padding='same' )(fe)
 	fe = LeakyReLU(alpha=0.2 )(fe)
 	
	fe = Flatten()(fe)
 	
	fe = Dropout(0.4 )(fe)
 	
	out_layer = Dense(1, activation='sigmoid' )(fe)
 	
	model = Model([in_image, in_label], out_layer)
 	
	opt = Adam(lr=0.0002, beta_1=0.5)
 	model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])
 	return model
df = pd.concat([df,pd.get_dummies(df['Target'], prefix='Target')],axis=1)
 df.head()
dataframes = [df,df_test]<feature_engineering>
dog = Input(( 12288,))
dogName = Input(( 10000,))
x = Dense(12288, activation='sigmoid' )(dogName)
 x = Reshape(( 2,12288,1))(concatenate([dog,x]))
x = Conv2D(1,(2,1),use_bias=False,name='conv' )(x)
 discriminated = Flatten()(x)
 discriminator = Model([dog,dogName], discriminated)
 discriminator.get_layer('conv' ).trainable = False
discriminator.get_layer('conv' ).set_weights([np.array([[[[-1.0 ]]],[[[1.0]]]])])
 discriminator.compile(optimizer='adam', loss='binary_crossentropy')
df.groupby('Target')['Target'].count()
T1p = 755.0/9557.0
T2p = 1597.0/9557.0
T3p = 1209.0/9557.0
T4p = 5996.0/9557.0
RT1p = 1.0/T1p
RT2p = 1.0/T2p
RT3p = 1.0/T3p
RT4p = 1.0/T4p
print(RT1p,RT2p,RT3p,RT4p)
 df['targetweight'] = 0.0
df.loc[df['Target']==1,'targetweight']=RT1p
df.loc[df['Target']==2,'targetweight']=RT2p
df.loc[df['Target']==3,'targetweight']=RT3p
df.loc[df['Target']==4,'targetweight']=RT4p<define_variables>
train_y =(imagesIn[:10000,:,:,:]/255.).reshape(( -1,12288))
train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1
zeros = np.zeros(( 10000,12288))
lr = 0.5
for k in range(5):
annealer = LearningRateScheduler(lambda x: lr)
 h = discriminator.fit([zeros,train_X], train_y, epochs = 20, batch_size=256, callbacks=[annealer], verbose=0)
 print('Epoch',(k+1)*10,'/50 - loss =',h.history['loss'][-1])
 if h.history['loss'][-1]<0.530: lr = 0.1
individual_variables = ['age','mobilephone','escolari','mobilephone','male','female','dis',
'instlevel1','instlevel2','instlevel3','instlevel4','instlevel5',
'instlevel6','instlevel7','instlevel8','instlevel9','estadocivil1',
'estadocivil2','estadocivil3','estadocivil4','estadocivil5','estadocivil6',
'estadocivil7','parentesco1','parentesco2','parentesco3','parentesco4',
'parentesco5','parentesco6','parentesco7','parentesco8','parentesco9',
'parentesco10','parentesco11','parentesco12']
for tdf in dataframes:
tdf['indeduindex25'] = 0
tdf.loc[tdf['age']>=25,'indeduindex25'] =(tdf['escolari'] / 15.0 +(7/18.0)) /2.0
tdf.loc[tdf['indeduindex25'].isna() ,'indeduindex25'] = 0
individual_variables.append('indeduindex25')
 tdf['agecentschool'] = tdf['age']-7
tdf['attendSchool'] = 0
tdf['attendSchool'] = np.where(tdf['escolari']>0,1,0)
 tdf['attendSchool'] = np.where(( tdf['age'] >=6)&(tdf['age'] <=13), tdf['attendSchool'],0)
 individual_variables.append('attendSchool')
 tdf['behindSchool'] = 0
tdf['behindSchool'] = np.where(tdf['agecentschool'] >tdf['escolari'],1,0)
 tdf['behindSchool'] = np.where(( tdf['age'] >=6)&(tdf['age'] <=13), tdf['behindSchool'],0)
 individual_variables.append('behindSchool')
 tdf['behindSchool18'] = 0
tdf['behindSchool18'] = np.where(tdf['agecentschool'] >tdf['escolari'],1,0)
 tdf['behindSchool18'] = np.where(( tdf['age'] >=6)&(tdf['age'] <=18), tdf['behindSchool18'],0)
 individual_variables.append('behindSchool18')
 tdf['portionofhh'] = 0
tdf['portionofhh'] = 1.0 / tdf['hhsize']
tdf.loc[tdf['parentesco1']==1,'portionofhh']=1.0
individual_variables.append('portionofhh')
 pd.set_option('display.max_columns', None)
 individual_variables = list(set(individual_variables))<define_variables>
def define_generator(latent_dim):
	model = Sequential()
	
	n_nodes = 128 * 7 * 7
	model.add(Dense(n_nodes, input_dim=latent_dim))
	model.add(LeakyReLU(alpha=0.2))
	model.add(Reshape(( 7, 7, 128)))
 	
	model.add(Conv2DTranspose(128,(4,4), strides=(2,2), padding='same'))
	model.add(LeakyReLU(alpha=0.2))
	
	model.add(Conv2DTranspose(128,(4,4), strides=(2,2), padding='same'))
	model.add(LeakyReLU(alpha=0.2))
	
	model.add(Conv2D(1,(7,7), activation='tanh', padding='same'))
	return model
household_variables = ['lugar1','lugar2','lugar3','lugar4','lugar5','lugar6','area1','area2',
'paredblolad','paredzocalo','paredpreb','pareddes','paredmad','paredzinc',
'paredfibras','paredother','pisomoscer','pisocemento','pisoother','pisonatur',
'pisonotiene','pisomadera','techozinc','techoentrepiso','techocane','techootro',
'cielorazo','abastaguadentro','abastaguafuera','abastaguano','public','planpri',
'noelec','coopele','sanitario1','sanitario2','sanitario3','sanitario5','sanitario6',
'energcocinar1','energcocinar2','energcocinar3','energcocinar4','elimbasu1','elimbasu2',
'elimbasu3','elimbasu4','elimbasu5','elimbasu6','epared1','epared2','epared3','etecho1',
'etecho2','etecho3','eviv1','eviv2','eviv3','tipovivi1','tipovivi2','tipovivi3','tipovivi4','tipovivi5',
'v18q1','qmobilephone','tamviv','tamhog','r4h1','r4h2','r4h3','r4m1','r4m2','r4m3','bedrooms', 'computer', 'hacapo', 'hacdor', 'hogar_adul', 'hogar_mayor', 'hogar_nin', 'r4t1',
'r4t2','r4t3', 'refrig', 'rooms', 'television', 'v14a', 'v18q']
for tdf in dataframes:
tdf.loc[tdf['v2a1'].isna() ,'v2a1'] = 0
tdf.loc[tdf['v18q1'].isna() ,'v18q1'] = 0
tdf.loc[tdf['qmobilephone'].isna() ,'qmobilephone'] = 0
conditions = [(tdf['v2a1']==0),(tdf['v2a1']!=0)]
choices = [0,np.log(tdf['v2a1']+.0001)]
tdf['lv2a1']=np.select(conditions, choices)
 household_variables.append('lv2a1')
 tdf['medhhage'] = tdf['age'].groupby(tdf['idhogar'] ).transform('median')
 household_variables.append('medhhage')
 tdf['eduindex'] = tdf[tdf['age']>=25]['escolari'].groupby(tdf['idhogar'] ).transform('mean')
 tdf['eduindex'] =(tdf['eduindex'] / 15.0 +(8.3/18.0)) /2.0
tdf['eduindex'] = tdf['eduindex'].groupby(tdf['idhogar'] ).transform('mean')
 tdf.loc[tdf['eduindex'].isna() ,'eduindex'] = 0
household_variables.append('eduindex')
 tdf['atleast8'] = 0
tdf['atleast8'] = tdf[tdf['escolari']>=8]['escolari'].groupby(tdf['idhogar'] ).transform('count')
 tdf['atleast8'] = tdf['atleast8'].groupby(tdf['idhogar'] ).transform('max')
 tdf['atleast8'] = np.where(tdf['atleast8']>=1, 1, 0)
 household_variables.append('atleast8')
 tdf['atleast8F'] = 0
tdf['atleast8F'] = tdf[(tdf['escolari']>=6)&(tdf['female']>=1)]['escolari'].groupby(tdf['idhogar'] ).transform('count')
 tdf['atleast8F'] = tdf['atleast8F'].groupby(tdf['idhogar'] ).transform('max')
 tdf['atleast8F'] = np.where(tdf['atleast8F']>=1, 1, 0)
 household_variables.append('atleast8F')
 tdf['nhhlt14'] = 0
tdf['nhhlt14'] = tdf[tdf['age']<=14]['age'].groupby(tdf['idhogar'] ).transform('count')
 tdf['nhhlt14'] = tdf['nhhlt14'].groupby(tdf['idhogar'] ).transform('max')
 tdf['nhhlt14'] = np.where(tdf['nhhlt14']>=1, tdf['nhhlt14'], 0)
 household_variables.append('nhhlt14')
 tdf['perhhlt14'] = 0.0
tdf['perhhlt14'] = tdf['nhhlt14'] / tdf['hhsize']
household_variables.append('perhhlt14')
 tdf['perschoolage'] = 0.0
tdf['perschoolage'] = tdf[(tdf['age'] >=6)&(tdf['age'] <=13)].groupby(tdf['idhogar'] ).transform('count')
 tdf['perschoolage'] = tdf['perschoolage'].groupby(tdf['idhogar'] ).transform('max')
 tdf['perschoolage'] = np.where(tdf['perschoolage']>=1, tdf['perschoolage'], 0)
 tdf['perschoolage'] = tdf['perschoolage'] / tdf['hhsize']
household_variables.append('perschoolage')
 tdf['cupara'] = tdf['hogar_adul']+(0.5*tdf['nhhlt14'])
 household_variables.append('cupara')
 tdf['assets2'] = 0
tdf.loc[(tdf['v18q']<=1)&(tdf['qmobilephone']<=1)&(tdf['computer']==0)&(tdf['television']==0),'assets2'] = 1
household_variables.append('assets2')
 tdf['assets3'] = 0
tdf.loc[(tdf['v18q']<=1)&(tdf['qmobilephone']>=1)&(tdf['computer']==0)&(tdf['television']==1),'assets3'] = 1
household_variables.append('assets3')
 tdf['assets3b'] = 0
tdf.loc[(tdf['v18q']<=1)&(tdf['qmobilephone']>=1)&(tdf['computer']==0)&(tdf['television']==0),'assets3b'] = 1
household_variables.append('assets3b')
 tdf['assets4'] = 0
tdf.loc[(tdf['v18q']>=1)&(tdf['qmobilephone']>=1)&(tdf['computer']==1)&(tdf['television']==1),'assets4'] = 1
household_variables.append('assets4')
 conditions = [(tdf['parentesco1']==1)&(tdf['estadocivil4']==1),(tdf['parentesco1']==1)&(tdf['estadocivil5']==1), 
(tdf['parentesco1']==1)&(tdf['estadocivil6']==1), 
(tdf['parentesco1']==1)&(tdf['estadocivil7']==1),(tdf['parentesco1']==0)]
choices = [1,1,1,1,0]
tdf['headmarital']=np.select(conditions, choices)
 tdf['headmarital'] = tdf['headmarital'].groupby(tdf['idhogar'] ).transform('count')
 tdf['headmarital'] = np.where(tdf['headmarital']>=1, 1, 0)
 household_variables.append('headmarital')
 conditions = [(tdf['hogar_adul']==0),(tdf['hogar_adul']>0)]
choices = [0, tdf['hogar_nin']/tdf['hogar_adul']]
tdf['radultchild']=np.select(conditions, choices)
 household_variables.append('radultchild')
 conditions = [(tdf['tamhog']==0),(tdf['tamhog']>0)]
choices = [0, tdf['bedrooms']/tdf['tamhog']]
tdf['bedperperson']=np.select(conditions, choices)
 household_variables.append('bedperperson')
 conditions = [(tdf['rooms']==0),(tdf['rooms']>0)]
choices = [0, tdf['bedrooms']/tdf['rooms']]
tdf['perbedrooms']=np.select(conditions, choices)
 household_variables.append('perbedrooms')
 conditions = [(tdf['rooms']==0),(tdf['rooms']>0)]
choices = [0, tdf['tamhog']/tdf['rooms']]
tdf['rperperson']=np.select(conditions, choices)
 household_variables.append('rperperson')
 conditions = [(tdf['rooms']==0),(tdf['rooms']>0)]
choices = [0, tdf['bedrooms']/tdf['rooms']]
tdf['roomstobedrooms']=np.select(conditions, choices)
 household_variables.append('roomstobedrooms')
 conditions = [(tdf['hogar_adul']==0),(tdf['hogar_adul']>0)]
choices = [0, tdf['lv2a1']/tdf['hogar_adul']]
tdf['rentpadult']=np.select(conditions, choices)
 household_variables.append('rentpadult')
 conditions = [(tdf['hogar_adul']==0),(tdf['hogar_adul']>0)]
choices = [0, tdf['qmobilephone']/tdf['hogar_adul']]
tdf['phoneperadult']=np.select(conditions, choices)
 household_variables.append('phoneperadult')
 conditions = [(tdf['hhsize']==0),(tdf['hhsize']>0)]
choices = [0, tdf['qmobilephone']/tdf['hhsize']]
tdf['phoneperhhsize']=np.select(conditions, choices)
 household_variables.append('phoneperhhsize')
 conditions = [(tdf['rooms']==0),(tdf['rooms']>0)]
choices = [0, tdf['lv2a1']/tdf['rooms']]
tdf['rentproom']=np.select(conditions, choices)
 household_variables.append('rentproom')
 conditions = [(tdf['dependency']=='yes'),(tdf['dependency']=='no'),(tdf['dependency']!='yes')&(tdf['dependency']!='no')]
choices = [1,0,0]
tdf['dphhyesno']=np.select(conditions, choices)
 household_variables.append('dphhyesno')
 conditions = [(tdf['dependency']=='yes'),(tdf['dependency']=='no'),(tdf['dependency']!='yes')&(tdf['dependency']!='no')]
choices = [0,0,tdf['dependency']]
tdf['dpnumeric']=np.select(conditions, choices)
 household_variables.append('dpnumeric')
 tdf['femhead'] = 0
tdf.loc[(tdf['parentesco1']==1)&(tdf['female']==1),'femhead'] = 1
tdf['femhead'] = tdf['femhead'].groupby(tdf['idhogar'] ).transform('count')
 tdf['femhead'] = np.where(tdf['femhead']>=1, 1, 0)
 household_variables.append('femhead')
 tdf['hhheadeduindex'] = 0.0
tdf['hhheadeduindex'] = tdf[tdf['parentesco1']==1]['escolari'].groupby(tdf['idhogar'] ).transform('mean')
 tdf['hhheadeduindex'] =(tdf['hhheadeduindex'] / 15.0 +(7.2/18.0)) /2.0
tdf.loc[tdf['hhheadeduindex'].isna() ,'hhheadeduindex'] = 0
tdf['hhheadeduindex'] = tdf['hhheadeduindex'].groupby(tdf['idhogar'] ).transform('mean')
 tdf.loc[tdf['hhheadeduindex'].isna() ,'hhheadeduindex'] = 0
household_variables.append('hhheadeduindex')
 tdf['hhheadage'] =0
tdf['hhheadage'] = tdf[tdf['parentesco1']==1]['age'].groupby(tdf['idhogar'] ).transform('mean')
 tdf['hhheadage'] = tdf['hhheadage'].groupby(tdf['idhogar'] ).transform('mean')
 tdf.loc[tdf['hhheadage'].isna() ,'hhheadage'] = 0
household_variables.append('hhheadage')
 tdf['misshead'] = 0
tdf['misshead'] = tdf['parentesco1'].groupby(tdf['idhogar'] ).transform('sum')
 tdf['misshead'] = np.where(tdf['misshead']==0, 1, 0)
 household_variables.append('misshead')
 conditions = [(tdf['edjefe']=='yes'),(tdf['edjefe']=='no'),(tdf['edjefe']!='yes')&(tdf['edjefe']!='no')]
choices = [0,1,0]
tdf['edjefemiss']=np.select(conditions, choices)
 household_variables.append('edjefemiss')
 conditions = [(tdf['edjefe']=='yes'),(tdf['edjefe']=='no'),(tdf['edjefe']!='yes')&(tdf['edjefe']!='no')]
choices = [0,0,tdf['edjefe']]
tdf['edjefeval']=np.select(conditions, choices)
 household_variables.append('edjefeval')
 conditions = [(tdf['edjefa']=='yes'),(tdf['edjefa']=='no'),(tdf['edjefa']!='yes')&(tdf['edjefa']!='no')]
choices = [0,1,0]
tdf['edjefamiss']=np.select(conditions, choices)
 household_variables.append('edjefamiss')
 conditions = [(tdf['edjefa']=='yes'),(tdf['edjefa']=='no'),(tdf['edjefa']!='yes')&(tdf['edjefa']!='no')]
choices = [0,0,tdf['edjefa']]
tdf['edjefaval']=np.select(conditions, choices)
 household_variables.append('edjefaval')
 tdf['region'] = tdf['region'] = tdf[locations].idxmax(axis = 1)
 tdf['r4h1permale'] = tdf['r4h1'] / tdf['r4h3']
tdf.loc[tdf['r4h1permale'].isna() ,'r4h1permale'] =0
household_variables.append('r4h1permale')
 tdf['r4m1perfemale'] = tdf['r4m1'] / tdf['r4m3']
tdf.loc[tdf['r4m1perfemale'].isna() ,'r4m1perfemale'] =0
household_variables.append('r4m1perfemale')
 tdf['r4h1pertotal'] = tdf['r4h1'] / tdf['hhsize']
household_variables.append('r4h1pertotal')
 tdf['r4m1pertotal'] = tdf['r4m1'] / tdf['hhsize']
household_variables.append('r4m1pertotal')
 tdf['r4t1pertotal'] = tdf['r4t1'] / tdf['hhsize']
household_variables.append('r4t1pertotal')
 tdf['cleanwater'] = 0
tdf.loc[(tdf['abastaguafuera']==1)|(tdf['abastaguano']==1),'cleanwater'] = 1
household_variables.append('cleanwater')
 tdf['improvedsan'] = 0
tdf.loc[(tdf['sanitario2']==1)|(tdf['sanitario3']==1)|(tdf['sanitario5']==1),'improvedsan'] = 1
household_variables.append('improvedsan')
 tdf['electricity'] = 0
tdf.loc[(tdf['public']==1)|(tdf['planpri']==1)|(tdf['coopele']==1),'electricity'] = 1
household_variables.append('electricity')
 tdf['flooring'] = 0
tdf.loc[(tdf['pisoother']==1)|(tdf['pisonatur']==1)|(tdf['pisonotiene']==1)|(tdf['pisomadera']==1),'flooring'] = 1
household_variables.append('flooring')
 tdf['wall'] = 0
tdf.loc[(tdf['paredzocalo']==1)|(tdf['pareddes']==1)|(tdf['paredfibras']==1)|(tdf['paredother']==1)|(tdf['paredmad']==1),'wall'] = 1
household_variables.append('wall')
 tdf['roof'] = 0
tdf.loc[(tdf['techocane']==1)|(tdf['techootro']==1)|(tdf['cielorazo']==0),'roof'] = 1
household_variables.append('roof')
 tdf['cooking'] = 0
tdf.loc[(tdf['energcocinar4']==1)|(tdf['energcocinar1']==1),'cooking'] = 1
household_variables.append('cooking')
 tdf['rubbish'] = 0
tdf.loc[(tdf['elimbasu3']==1)|(tdf['elimbasu4']==1)|(tdf['elimbasu5']==1)|(tdf['elimbasu6']==1),'rubbish'] = 1
household_variables.append('rubbish')
 tdf['condition'] = 0
tdf.loc[(tdf['epared1']==1)&(tdf['etecho1']==1)&(tdf['eviv1']==1),'condition'] = 1
household_variables.append('condition')
 tdf['condition2'] = 0
tdf.loc[(tdf['epared2']==1)&(tdf['etecho2']==1)&(tdf['eviv2']==1),'condition2'] = 1
household_variables.append('condition2')
 tdf['condition3'] = 0
tdf.loc[(tdf['epared3']==1)&(tdf['etecho3']==1)&(tdf['eviv3']==1),'condition3'] = 1
household_variables.append('condition3')
 household_variables = list(set(household_variables)) 
<feature_engineering>
def define_generator(latent_dim, n_classes=120):
	
	in_label = Input(shape=(1,))
	
	li = Embedding(n_classes, 50 )(in_label)
 	
	n_nodes = 7 * 7
	li = Dense(n_nodes )(li)
 	
	li = Reshape(( 7, 7, 1))(li)
 	
	in_lat = Input(shape=(latent_dim,))
	
	n_nodes = 128 * 7 * 7
	gen = Dense(n_nodes )(in_lat)
 	gen = LeakyReLU(alpha=0.2 )(gen)
 	gen = Reshape(( 7, 7, 128))(gen)
 	
	merge = Concatenate()([gen, li])
 	
	gen = Conv2DTranspose(128,(4,4), strides=(2,2), padding='same' )(merge)
 	gen = LeakyReLU(alpha=0.2 )(gen)
 	
	gen = Conv2DTranspose(128,(4,4), strides=(2,2), padding='same' )(gen)
 	gen = LeakyReLU(alpha=0.2 )(gen)
 	
	out_layer = Conv2D(1,(7,7), activation='tanh', padding='same' )(gen)
 	
	model = Model([in_lat, in_label], out_layer)
 	return model
df_regional = df.copy()
df_test_regional = df_test.copy()
df_test_regional["Target"]=0
df_test_regional["Target0"]=0
df_all = pd.concat([df_regional,df_test_regional],sort=False)
 df_all.index = range(0,len(df_all['idhogar']))
regional_variables = []
df_all['regionalMeanLogRent'] = 0.0
df_all['regionalMeanLogRent'] = df_all[df_all['lv2a1']>0][['lv2a1','idhogar','region']].groupby(['idhogar','region'] ).transform('mean' ).groupby(df_all['region'] ).transform('mean')
 df_all.loc[df_all['regionalMeanLogRent'].isna() ,'regionalMeanLogRent'] = 0
regional_variables.append('regionalMeanLogRent')
 df_all['rmedianage'] = df_all[['age','idhogar','region']].groupby(['region'] ).transform('median')
 regional_variables.append('rmedianage')
 df_all['reduindex'] = df_all[['eduindex','idhogar','region']].groupby(['idhogar','region'] ).transform('mean' ).groupby(df_all['region'] ).transform('mean')
 regional_variables.append('reduindex')
 df_all['rnhhlt14'] = df_all[['nhhlt14','idhogar','region']].groupby(['idhogar','region'] ).transform('mean' ).groupby(df_all['region'] ).transform('mean')
 regional_variables.append('rnhhlt14')
 df_all['rhogar_total'] = df_all[['hogar_total','idhogar','region']].groupby(['idhogar','region'] ).transform('mean' ).groupby(df_all['region'] ).transform('mean')
 regional_variables.append('rhogar_total')
 df_all['rbedrooms'] = df_all[['bedrooms','idhogar','region']].groupby(['idhogar','region'] ).transform('mean' ).groupby(df_all['region'] ).transform('mean')
 regional_variables.append('rbedrooms')
 df_all['urbanhouseholds']=df_all[['area1','idhogar','region']].groupby(['idhogar','region'] ).transform('mean' ).groupby(df_all['region'] ).transform('mean')
 regional_variables.append('urbanhouseholds')
 for fld in regional_variables:
tempPiv = pd.pivot_table(df_all, values=fld, index=['region'], aggfunc=np.mean)
 tempPiv.columns = [fld]
df = df.merge(tempPiv,on='region',how='left')
 del tempPiv
for fld in regional_variables:
tempPiv = pd.pivot_table(df_all, values=fld, index=['region'], aggfunc=np.mean)
 tempPiv.columns = [fld]
df_test = df_test.merge(tempPiv,on='region',how='left')
 del tempPiv<data_type_conversions>
discriminator.trainable=False
gan_input = Input(shape=(10000,))
x = generator(gan_input)
 gan_output = discriminator(x)
 gan = Model(gan_input, gan_output)
 gan.get_layer('model_1' ).get_layer('conv' ).set_weights([np.array([[[[-1 ]]],[[[255.]]]])])
 gan.compile(optimizer=Adam(5), loss='mean_squared_error')
for tdf in dataframes:
print(tdf.columns[tdf.isna().any() ].tolist())
 for f in tdf.columns[tdf.isna().any() ].tolist() :
if f in regional_variables:
print(f)
 print(tdf.columns[tdf.isnull().any() ].tolist())
 for f in tdf.columns[tdf.isnull().any() ].tolist() :
if f in regional_variables:
print(f )<import_modules>
train = np.zeros(( 10000,10000))
for i in range(10000): train[i,i] = 1
zeros = np.zeros(( 10000,12288))
lr = 5.
 for k in range(50):
annealer = LearningRateScheduler(lambda x: lr)
 h = gan.fit(train, zeros, epochs = 1, batch_size=256, callbacks=[annealer], verbose=0)
 if(k<10)|(k%5==4):
print('Epoch',(k+1)*10,'/500 - loss =',h.history['loss'][-1])
 if h.history['loss'][-1] < 25: lr = 1.
 if h.history['loss'][-1] < 1.5: lr = 0.5
if k<10:
plt.figure(figsize=(15,3))
for j in range(5):
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 img = generator.predict(xx.reshape(( -1,10000)))[0].reshape(( -1,64,64,3))
img = Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()
combo = individual_variables+household_variables+regional_variables
combo = list(set(combo))
index_individual_variables = list(range(0,len(individual_variables)))
 index_household_variables = list(range(len(individual_variables),len(individual_variables)+len(household_variables)))
 index_indiv_and_hh_variables = index_individual_variables+index_household_variables<define_variables>
class DogGenerator:
index = 0
def getDog(self,seed):
xx = np.zeros(( 10000))
xx[self.index] = 0.999
xx[np.random.randint(10000)] = 0.001
img = generator.predict(xx.reshape(( -1,10000)))[0].reshape(( 64,64,3))
self.index =(self.index+1)%10000
return Image.fromarray(img.astype('uint8'))
keeplst = []
indvmodels = {}
for t in [1,2,3,4]:
params = {'n_estimators': 500,
'max_depth':None,
'min_samples_split':2,
'min_samples_leaf':90,
'max_features':'auto',
'loss':'deviance',
"learning_rate":.01,
"subsample":.5}
currentTarget = "Target_"+str(t)
 print(currentTarget)
 currentTarget_v = t
df_sub = df[df['parentesco1']==1][combo+[currentTarget,"idhogarnum",'targetweight']].copy()
X = df_sub[combo].values
y = df_sub[currentTarget].values.ravel()
weights = df_sub['targetweight'].values.ravel()
Y_strat = df_sub['idhogarnum'].values
gss = GroupShuffleSplit(test_size=.1, n_splits=1,random_state=314)
 for train_index, test_index in gss.split(X, y, groups=Y_strat):
break
X_train, X_test = X[train_index],X[test_index]
y_train, y_test = y[train_index],y[test_index]
w_train = weights[train_index]
strat_train = Y_strat[train_index]
print(X_train.shape)
 print(X_test.shape)
 reg = GradientBoostingClassifier(**params)
 reg.fit(X_train,y_train,sample_weight=w_train)
 indvmodels[t] = reg
y_pred = reg.predict(X_test)
 print(f1_score(y_test, y_pred, average='macro'))
print(classification_report(y_test, y_pred))
fi = list(zip(range(0,len(reg.feature_importances_)) ,reg.feature_importances_))
fi.sort(key=lambda x: x[1])
 vals = [x[1] for x in fi]
for i,important in fi:
if important >=np.percentile(reg.feature_importances_, 75):
keeplst.append(combo[i])
 print("%s: %s"%(combo[i],important))
probabilities = pd.DataFrame(reg.predict_proba(X),columns=["P0","P1"])
 df[currentTarget+"_p"]=0.1
df.loc[df['parentesco1']==1,currentTarget+"_p"]= probabilities["P1"].values
print(list(set(keeplst)) )<feature_engineering>
class KernelEvalException(Exception):
pass
model_params = {
'Inception': {
'name': 'Inception',
'imsize': 64,
'output_layer': 'Pretrained_Net/pool_3:0',
'input_layer': 'Pretrained_Net/ExpandDims:0',
'output_shape': 2048,
'cosine_distance_eps': 0.1
}
}
def create_model_graph(pth):

 with tf.gfile.FastGFile(pth, 'rb')as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
 _ = tf.import_graph_def(graph_def, name='Pretrained_Net')
 def _get_model_layer(sess, model_name):
layername = model_params[model_name]['output_layer']
layer = sess.graph.get_tensor_by_name(layername)
 ops = layer.graph.get_operations()
for op_idx, op in enumerate(ops):
for o in op.outputs:
shape = o.get_shape()
if shape._dims != []:
shape = [s.value for s in shape]
new_shape = []
for j, s in enumerate(shape):
if s == 1 and j == 0:
new_shape.append(None)
 else:
new_shape.append(s)
 o.__dict__['_shape_val'] = tf.TensorShape(new_shape)
 return layer
def get_activations(images, sess, model_name, batch_size=50, verbose=False):

 inception_layer = _get_model_layer(sess, model_name)
 n_images = images.shape[0]
if batch_size > n_images:
print("warning: batch size is bigger than the data size.setting batch size to data size")
 batch_size = n_images
n_batches = n_images//batch_size + 1
pred_arr = np.empty(( n_images,model_params[model_name]['output_shape']))
for i in tqdm(range(n_batches)) :
if verbose:
print("\rPropagating batch %d/%d" %(i+1, n_batches), end="", flush=True)
 start = i*batch_size
if start+batch_size < n_images:
end = start+batch_size
else:
end = n_images
batch = images[start:end]
pred = sess.run(inception_layer, {model_params[model_name]['input_layer']: batch})
 pred_arr[start:end] = pred.reshape(-1,model_params[model_name]['output_shape'])
 if verbose:
print(" done")
 return pred_arr
def normalize_rows(x: np.ndarray):

 return np.nan_to_num(x/np.linalg.norm(x, ord=2, axis=1, keepdims=True))
def cosine_distance(features1, features2):
features1_nozero = features1[np.sum(features1, axis=1)!= 0]
features2_nozero = features2[np.sum(features2, axis=1)!= 0]
norm_f1 = normalize_rows(features1_nozero)
 norm_f2 = normalize_rows(features2_nozero)
 d = 1.0-np.abs(np.matmul(norm_f1, norm_f2.T))
print('d.shape=',d.shape)
 print('np.min(d, axis=1 ).shape=',np.min(d, axis=1 ).shape)
 mean_min_d = np.mean(np.min(d, axis=1))
print('distance=',mean_min_d)
 return mean_min_d
def distance_thresholding(d, eps):
if d < eps:
return d
else:
return 1
def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):

 mu1 = np.atleast_1d(mu1)
 mu2 = np.atleast_1d(mu2)
 sigma1 = np.atleast_2d(sigma1)
 sigma2 = np.atleast_2d(sigma2)
 assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"
diff = mu1 - mu2
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
 if not np.isfinite(covmean ).all() :
msg = "fid calculation produces singular product; adding %s to diagonal of cov estimates" % eps
warnings.warn(msg)
 offset = np.eye(sigma1.shape[0])* eps
covmean = linalg.sqrtm(( sigma1 + offset ).dot(sigma2 + offset))
if np.iscomplexobj(covmean):
if not np.allclose(np.diagonal(covmean ).imag, 0, atol=1e-3):
m = np.max(np.abs(covmean.imag))
raise ValueError("Imaginary component {}".format(m))
covmean = covmean.real
print('covmean.shape=',covmean.shape)
 tr_covmean = np.trace(covmean)
 return diff.dot(diff)+ np.trace(sigma1)+ np.trace(sigma2)- 2 * tr_covmean
def calculate_activation_statistics(images, sess, model_name, batch_size=50, verbose=False):

 act = get_activations(images, sess, model_name, batch_size, verbose)
 mu = np.mean(act, axis=0)
 sigma = np.cov(act, rowvar=False)
 return mu, sigma, act
def _handle_path_memorization(path, sess, model_name, is_checksize, is_check_png):
path = pathlib.Path(path)
 files = list(path.glob('*.jpg')) + list(path.glob('*.png'))
imsize = model_params[model_name]['imsize']
x = np.array([np.array(img_read_checks(fn, imsize, is_checksize, imsize, is_check_png)) for fn in files])
 m, s, features = calculate_activation_statistics(x, sess, model_name)
 del x
return m, s, features
def img_read_checks(filename, resize_to, is_checksize=False, check_imsize = 64, is_check_png = False):
im = Image.open(str(filename))
if is_checksize and im.size !=(check_imsize,check_imsize):
raise KernelEvalException('The images are not of size '+str(check_imsize))
if is_check_png and im.format != 'PNG':
raise KernelEvalException('Only PNG images should be submitted.')
 if resize_to is None:
return im
else:
return im.resize(( resize_to,resize_to),Image.ANTIALIAS)
 def calculate_kid_given_paths(paths, model_name, model_path, feature_path=None, mm=[], ss=[], ff=[]):

 tf.reset_default_graph()
create_model_graph(str(model_path))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
 m1, s1, features1 = _handle_path_memorization(paths[0], sess, model_name, is_checksize = True, is_check_png = True)
 if len(mm)!= 0:
m2 = mm
s2 = ss
features2 = ff
elif feature_path is None:
m2, s2, features2 = _handle_path_memorization(paths[1], sess, model_name, is_checksize = False, is_check_png = False)
 else:
with np.load(feature_path)as f:
m2, s2, features2 = f['m'], f['s'], f['features']
print('m1,m2 shape=',(m1.shape,m2.shape),'s1,s2=',(s1.shape,s2.shape))
print('starting calculating FID')
 fid_value = calculate_frechet_distance(m1, s1, m2, s2)
 print('done with FID, starting distance calculation')
 distance = cosine_distance(features1, features2)
 return fid_value, distance, m2, s2, features2
<split><EOS> 
if ComputeLB:
with zipfile.ZipFile(".. /working/images.zip","r")as z:
z.extractall(".. /tmp/images2/")
 m2 = []; s2 =[]; f2 = []
user_images_unzipped_path = '.. /tmp/images2/'
images_path = [user_images_unzipped_path,'.. /input/generative-dog-images/all-dogs/all-dogs/']
public_path = '.. /input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'
fid_epsilon = 10e-15
fid_value_public, distance_public, m2, s2, f2 = calculate_kid_given_paths(images_path, 'Inception', public_path, mm=m2, ss=s2, ff=f2)
 distance_public = distance_thresholding(distance_public, model_params['Inception']['cosine_distance_eps'])
 print("FID_public: ", fid_value_public, "distance_public: ", distance_public, "multiplied_public: ",
fid_value_public /(distance_public + fid_epsilon))
! rm -r.. /tmp