Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<feature_engineering>
%matplotlib inline
def get_features(path):
try:
st = []
img = Image.open(path)
 img = img.resize(( 100,int(img.size[1]/(img.size[0]/100))), Image.ANTIALIAS)
 img = img.crop(( 0, 0, 64, 64))
im_stats_ = ImageStat.Stat(img)
 st += im_stats_.sum
st += im_stats_.mean
st += im_stats_.rms
st += im_stats_.var
st += im_stats_.stddev
img = np.array(img)
 m, s = cv2.meanStdDev(img)
 st += list(m)
 st += list(s)
 st += [cv2.Laplacian(img, cv2.CV_64F ).var() ]
st += [cv2.Sobel(img,cv2.CV_64F,1,0,ksize=5 ).var() ]
st += [cv2.Sobel(img,cv2.CV_64F,0,1,ksize=5 ).var() ]
img = img[:,:,:3]
st += [scipy.stats.kurtosis(img[:,:,0].ravel())]
st += [scipy.stats.kurtosis(img[:,:,1].ravel())]
st += [scipy.stats.kurtosis(img[:,:,2].ravel())]
st += [scipy.stats.skew(img[:,:,0].ravel())]
st += [scipy.stats.skew(img[:,:,1].ravel())]
st += [scipy.stats.skew(img[:,:,2].ravel())]
except:
print(path)
 return [path, st]
def normalize_img(paths):
imf_d = {}
p = Pool(cpu_count())
 ret = p.map(get_features, paths)
 for i in range(len(ret)) :
imf_d[ret[i][0]] = ret[i][1]
ret = []
fdata = [imf_d[f] for f in paths]
return pd.DataFrame(fdata)
 dog_bytes = pd.DataFrame(glob.glob('.. /input/all-dogs/all-dogs/**'), columns=['Path'])
 dog_bytes = pd.concat(( dog_bytes, normalize_img(dog_bytes.Path.values)) , axis=1 )
ind_train_data = AddFeatures(ind_train_data, ind_bool, ['mean','sum'], ind_train_groupobj)
 funcs = ['mean','min','max','median','sum','nunique']
ind_train_data = AddFeatures(ind_train_data, ind_non_bool, funcs, ind_train_groupobj )<create_dataframe>
labels = pd.DataFrame(glob.glob('.. /input/annotation/Annotation/**/**'), columns=['Path'])
 labels['Labels'] = labels['Path'].map(lambda x: x.split('/')[4].split('-')[1])
 labels['FileName'] = labels['Path'].map(lambda x: x.split('/')[-1] + '.jpg')
 labels = {f:l for f,l in labels[['FileName', 'Labels']].values}
dog_bytes['FileName'] = dog_bytes['Path'].map(lambda x: x.split('/')[-1])
 dog_bytes['Labels'] = dog_bytes['FileName'].map(labels)
 dog_bytes.head()
ind_test_groupobj = ind_test_df.groupby(by='idhogar')
 ind_test_data = pd.DataFrame({'idhogar':ind_test_df.idhogar.unique() })
 ind_test_data = AddFeatures(ind_test_data, ind_bool, ['mean','sum'], ind_test_groupobj)
 ind_test_data = AddFeatures(ind_test_data, ind_non_bool, funcs, ind_test_groupobj )<merge>
dog_bytes['Group'] = cluster.KMeans(n_clusters=400, random_state=4 ).fit_predict(dog_bytes[list(range(30)) ])
 dog_bytes['Group'].value_counts() [:5]
<import_modules><EOS> 
def sim_img(path):
img = Image.open(path ).convert('RGB')
 img = img.resize(( 100,int(img.size[1]/(img.size[0]/100))), Image.ANTIALIAS)
 img = img.crop(( 0, 0, 64, 64))
return img
samples = []
for i in range(400):
g = dog_bytes[dog_bytes['Group'] == i]
p = g['Path'].values
for i in range(0,len(p)-2, 2):
samples.append([p[i],p[i+1]])
 if len(samples)> 11_000: break
for i in range(0,len(samples)-1):
samples.append([samples[i][0],samples[i+1][0]])
 if len(samples)> 11_000: break
print(len(samples))