Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<save_to_csv>
ComputeLB = True
DogsOnly = True
ROOT = '.. /input/generative-dog-images/'
if not ComputeLB: ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 namesIn.append(breed)
 idxIn += 1
idx = np.arange(idxIn)
 np.random.shuffle(idx)
 imagesIn = imagesIn[idx,:,:,:]
namesIn = np.array(namesIn)[idx]
else:
x = np.random.choice(np.arange(20579),10000)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]
h = img.size[1]
sz = np.min(( w,h))
a=0; b=0
if w<h: b =(h-sz)//2
else: a =(w-sz)//2
img = img.crop(( 0+a, 0+b, sz+a, sz+b))
img = img.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 if idxIn%1000==0: print(idxIn)
 idxIn += 1
x = np.random.randint(0,idxIn,25)
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = Image.fromarray(imagesIn[x[k*5+j],:,:,:].astype('uint8'))
plt.axis('off')
 if not DogsOnly: plt.title(namesIn[x[k*5+j]],fontsize=11)
 else: plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)
 plt.imshow(img)
 plt.show()
X_fin = df_test[combo].values
df_test['TargetP'] = reg.predict(X_fin)+1
print(df_test.TargetP)
 df_output = pd.DataFrame({"Id":df_test['Id'],"Target":df_test['TargetP']})
 df_output.to_csv("outputTargetPropensity.csv",index=False )<prepare_output>
from keras.models import Model
from keras.layers import Input, Dense, Conv2D, Reshape, Flatten, concatenate, UpSampling2D
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
from keras.optimizers import SGD, Adam
X_fin = df_test[combo].values
y_indiv = np.zeros([X_fin.shape[0], 4], dtype=float)
 for t in [1,2,3,4]:
reg = indvmodels[t]
df_prob = pd.DataFrame(reg.predict_proba(X_fin),columns=['B0','B1'])
 y_indiv[:, t-1] = df_prob['B1'].values
del df_prob
y_pred= np.zeros(X_fin.shape[0])
 for i in range(X_fin.shape[0]):
y_pred[i]=np.argmax(y_indiv[i])
 y_pred = np.array(y_pred,dtype=int)
 df_test['TargetI'] = y_pred+1
print(df_test.TargetI)
 df_output = pd.DataFrame({"Id":df_test['Id'],"Target":df_test['TargetI']})
 df_output.to_csv("outputTargetIndiv.csv",index=False )<set_options>
train_y =(imagesIn[:10000,:,:,:]/255.).reshape(( -1,12288))
train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1
zeros = np.zeros(( 10000,12288))
lr = 0.5
for k in range(5):
annealer = LearningRateScheduler(lambda x: lr)
 h = discriminator.fit([zeros,train_X], train_y, epochs = 10, batch_size=256, callbacks=[annealer], verbose=0)
 print('Epoch',(k+1)*10,'/30 - loss =',h.history['loss'][-1])
 if h.history['loss'][-1]<0.533: lr = 0.1
%matplotlib inline
warnings.filterwarnings("ignore" )<categorify>
del train_X, train_y, imagesIn
def encode_data(df):
df['idhogar'] = LabelEncoder().fit_transform(df['idhogar'])
 def feature_importance(forest, X_train):
ranked_list = []
importances = forest.feature_importances_
indices = np.argsort(importances)[::-1]
print("Feature ranking:")
 for f in range(X_train.shape[1]):
print("%d.feature %d(%f)" %(f + 1, indices[f], importances[indices[f]])+ " - " + X_train.columns[indices[f]])
 ranked_list.append(X_train.columns[indices[f]])
 return ranked_list<feature_engineering>
BadMemory = True
if BadMemory:
seed = Input(( 10000,))
x = Dense(2048, activation='elu' )(seed)
 x = Reshape(( 8,8,32))(x)
 x = Conv2D(128,(3, 3), activation='elu', padding='same' )(x)
 x = UpSampling2D(( 2, 2))(x)
 x = Conv2D(64,(3, 3), activation='elu', padding='same' )(x)
 x = UpSampling2D(( 2, 2))(x)
 x = Conv2D(32,(3, 3), activation='elu', padding='same' )(x)
 x = UpSampling2D(( 2, 2))(x)
 x = Conv2D(3,(3, 3), activation='linear', padding='same' )(x)
 generated = Flatten()(x)
 else:
seed = Input(( 10000,))
generated = Dense(12288, activation='linear' )(seed)
 generator = Model(seed, [generated,Reshape(( 10000,))(seed)])
 generator.summary()
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop(['Id'], axis=1, inplace=True)
 return df<categorify>
discriminator.trainable=False
gan_input = Input(shape=(10000,))
x = generator(gan_input)
 gan_output = discriminator(x)
 gan = Model(gan_input, gan_output)
 gan.get_layer('model_1' ).get_layer('conv' ).set_weights([np.array([[[[-1 ]]],[[[255.]]]])])
 gan.compile(optimizer=Adam(5), loss='mean_squared_error')
 gan.summary()
def convert_OHE2LE(df):
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print('The OHE in {} is incomplete.A new column will be added before label encoding' 
.format(s_))
col_dummy = s_+'_dummy'
tmp_df[col_dummy] =(tmp_df[cols_s_].sum(axis=1)== 0 ).astype(np.int8)
 cols_s_.append(col_dummy)
 sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print("The category completion did not work")
 tmp_cat = tmp_df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df<load_from_csv>
train = np.zeros(( 10000,10000))
for i in range(10000): train[i,i] = 1
zeros = np.zeros(( 10000,12288))
ep = 1; it = 9
if BadMemory: lr = 0.005
else: lr = 5.
 for k in range(it):
annealer = LearningRateScheduler(lambda x: lr)
 h = gan.fit(train, zeros, epochs = ep, batch_size=256, callbacks=[annealer], verbose=0)
 print('Epoch',(k+1),'/'+str(it)+' - loss =',h.history['loss'][-1])
 plt.figure(figsize=(15,3))
for j in range(5):
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 img = generator.predict(xx.reshape(( -1,10000)))[0].reshape(( -1,64,64,3))
img = Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()
if BadMemory:
ep *= 2
if ep>=32: lr = 0.001
if ep>256: ep = 256
else:
if h.history['loss'][-1] < 25: lr = 1.
 if h.history['loss'][-1] < 1.5: lr = 0.5
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test_ids = test.Id<categorify>
class DogGenerator:
index = 0
def getDog(self,seed):
xx = np.zeros(( 10000))
xx[self.index] = 0.70
xx[np.random.randint(10000)] = 0.30
img = generator.predict(xx.reshape(( -1,10000)))[0].reshape(( 64,64,3))
self.index =(self.index+1)%10000
return Image.fromarray(img.astype('uint8'))
def process_df(df_):
encode_data(df_)
 return do_features(df_)
 train = process_df(train)
 test = process_df(test )<feature_engineering>
class KernelEvalException(Exception):
pass
model_params = {
'Inception': {
'name': 'Inception',
'imsize': 64,
'output_layer': 'Pretrained_Net/pool_3:0',
'input_layer': 'Pretrained_Net/ExpandDims:0',
'output_shape': 2048,
'cosine_distance_eps': 0.1
}
}
def create_model_graph(pth):

 with tf.gfile.FastGFile(pth, 'rb')as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
 _ = tf.import_graph_def(graph_def, name='Pretrained_Net')
 def _get_model_layer(sess, model_name):
layername = model_params[model_name]['output_layer']
layer = sess.graph.get_tensor_by_name(layername)
 ops = layer.graph.get_operations()
for op_idx, op in enumerate(ops):
for o in op.outputs:
shape = o.get_shape()
if shape._dims != []:
shape = [s.value for s in shape]
new_shape = []
for j, s in enumerate(shape):
if s == 1 and j == 0:
new_shape.append(None)
 else:
new_shape.append(s)
 o.__dict__['_shape_val'] = tf.TensorShape(new_shape)
 return layer
def get_activations(images, sess, model_name, batch_size=50, verbose=False):

 inception_layer = _get_model_layer(sess, model_name)
 n_images = images.shape[0]
if batch_size > n_images:
print("warning: batch size is bigger than the data size.setting batch size to data size")
 batch_size = n_images
n_batches = n_images//batch_size + 1
pred_arr = np.empty(( n_images,model_params[model_name]['output_shape']))
for i in tqdm(range(n_batches)) :
if verbose:
print("\rPropagating batch %d/%d" %(i+1, n_batches), end="", flush=True)
 start = i*batch_size
if start+batch_size < n_images:
end = start+batch_size
else:
end = n_images
batch = images[start:end]
pred = sess.run(inception_layer, {model_params[model_name]['input_layer']: batch})
 pred_arr[start:end] = pred.reshape(-1,model_params[model_name]['output_shape'])
 if verbose:
print(" done")
 return pred_arr
def normalize_rows(x: np.ndarray):

 return np.nan_to_num(x/np.linalg.norm(x, ord=2, axis=1, keepdims=True))
def cosine_distance(features1, features2):
features1_nozero = features1[np.sum(features1, axis=1)!= 0]
features2_nozero = features2[np.sum(features2, axis=1)!= 0]
norm_f1 = normalize_rows(features1_nozero)
 norm_f2 = normalize_rows(features2_nozero)
 d = 1.0-np.abs(np.matmul(norm_f1, norm_f2.T))
print('d.shape=',d.shape)
 print('np.min(d, axis=1 ).shape=',np.min(d, axis=1 ).shape)
 mean_min_d = np.mean(np.min(d, axis=1))
print('distance=',mean_min_d)
 return mean_min_d
def distance_thresholding(d, eps):
if d < eps:
return d
else:
return 1
def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):

 mu1 = np.atleast_1d(mu1)
 mu2 = np.atleast_1d(mu2)
 sigma1 = np.atleast_2d(sigma1)
 sigma2 = np.atleast_2d(sigma2)
 assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"
diff = mu1 - mu2
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
 if not np.isfinite(covmean ).all() :
msg = "fid calculation produces singular product; adding %s to diagonal of cov estimates" % eps
warnings.warn(msg)
 offset = np.eye(sigma1.shape[0])* eps
covmean = linalg.sqrtm(( sigma1 + offset ).dot(sigma2 + offset))
if np.iscomplexobj(covmean):
if not np.allclose(np.diagonal(covmean ).imag, 0, atol=1e-3):
m = np.max(np.abs(covmean.imag))
raise ValueError("Imaginary component {}".format(m))
covmean = covmean.real
print('covmean.shape=',covmean.shape)
 tr_covmean = np.trace(covmean)
 return diff.dot(diff)+ np.trace(sigma1)+ np.trace(sigma2)- 2 * tr_covmean
def calculate_activation_statistics(images, sess, model_name, batch_size=50, verbose=False):

 act = get_activations(images, sess, model_name, batch_size, verbose)
 mu = np.mean(act, axis=0)
 sigma = np.cov(act, rowvar=False)
 return mu, sigma, act
def _handle_path_memorization(path, sess, model_name, is_checksize, is_check_png):
path = pathlib.Path(path)
 files = list(path.glob('*.jpg')) + list(path.glob('*.png'))
imsize = model_params[model_name]['imsize']
x = np.array([np.array(img_read_checks(fn, imsize, is_checksize, imsize, is_check_png)) for fn in files])
 m, s, features = calculate_activation_statistics(x, sess, model_name)
 del x
return m, s, features
def img_read_checks(filename, resize_to, is_checksize=False, check_imsize = 64, is_check_png = False):
im = Image.open(str(filename))
if is_checksize and im.size !=(check_imsize,check_imsize):
raise KernelEvalException('The images are not of size '+str(check_imsize))
if is_check_png and im.format != 'PNG':
raise KernelEvalException('Only PNG images should be submitted.')
 if resize_to is None:
return im
else:
return im.resize(( resize_to,resize_to),Image.ANTIALIAS)
 def calculate_kid_given_paths(paths, model_name, model_path, feature_path=None, mm=[], ss=[], ff=[]):

 tf.reset_default_graph()
create_model_graph(str(model_path))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
 m1, s1, features1 = _handle_path_memorization(paths[0], sess, model_name, is_checksize = True, is_check_png = True)
 if len(mm)!= 0:
m2 = mm
s2 = ss
features2 = ff
elif feature_path is None:
m2, s2, features2 = _handle_path_memorization(paths[1], sess, model_name, is_checksize = False, is_check_png = False)
 else:
with np.load(feature_path)as f:
m2, s2, features2 = f['m'], f['s'], f['features']
print('m1,m2 shape=',(m1.shape,m2.shape),'s1,s2=',(s1.shape,s2.shape))
print('starting calculating FID')
 fid_value = calculate_frechet_distance(m1, s1, m2, s2)
 print('done with FID, starting distance calculation')
 distance = cosine_distance(features1, features2)
 return fid_value, distance, m2, s2, features2
<concatenate><EOS> 
if ComputeLB:
with zipfile.ZipFile(".. /working/images.zip","r")as z:
z.extractall(".. /tmp/images2/")
 m2 = []; s2 =[]; f2 = []
user_images_unzipped_path = '.. /tmp/images2/'
images_path = [user_images_unzipped_path,'.. /input/generative-dog-images/all-dogs/all-dogs/']
public_path = '.. /input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'
fid_epsilon = 10e-15
fid_value_public, distance_public, m2, s2, f2 = calculate_kid_given_paths(images_path, 'Inception', public_path, mm=m2, ss=s2, ff=f2)
 distance_public = distance_thresholding(distance_public, model_params['Inception']['cosine_distance_eps'])
 print("FID_public: ", fid_value_public, "distance_public: ", distance_public, "multiplied_public: ",
fid_value_public /(distance_public + fid_epsilon))
! rm -r.. /tmp