Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<sort_values>
%matplotlib inline
def get_features(path):
try:
st = []
img = Image.open(path)
 img = img.resize(( 100,int(img.size[1]/(img.size[0]/100))), Image.ANTIALIAS)
 img = img.crop(( 0, 0, 64, 64))
im_stats_ = ImageStat.Stat(img)
 st += im_stats_.sum
st += im_stats_.mean
st += im_stats_.rms
st += im_stats_.var
st += im_stats_.stddev
img = np.array(img)
 m, s = cv2.meanStdDev(img)
 st += list(m)
 st += list(s)
 st += [cv2.Laplacian(img, cv2.CV_64F ).var() ]
st += [cv2.Sobel(img,cv2.CV_64F,1,0,ksize=5 ).var() ]
st += [cv2.Sobel(img,cv2.CV_64F,0,1,ksize=5 ).var() ]
img = img[:,:,:3]
st += [scipy.stats.kurtosis(img[:,:,0].ravel())]
st += [scipy.stats.kurtosis(img[:,:,1].ravel())]
st += [scipy.stats.kurtosis(img[:,:,2].ravel())]
st += [scipy.stats.skew(img[:,:,0].ravel())]
st += [scipy.stats.skew(img[:,:,1].ravel())]
st += [scipy.stats.skew(img[:,:,2].ravel())]
except:
print(path)
 return [path, st]
def normalize_img(paths):
imf_d = {}
p = Pool(cpu_count())
 ret = p.map(get_features, paths)
 for i in range(len(ret)) :
imf_d[ret[i][0]] = ret[i][1]
ret = []
fdata = [imf_d[f] for f in paths]
return pd.DataFrame(fdata)
 dog_bytes = pd.DataFrame(glob.glob('.. /input/all-dogs/all-dogs/**'), columns=['Path'])
 dog_bytes = pd.concat(( dog_bytes, normalize_img(dog_bytes.Path.values)) , axis=1 )
total = data.isnull().sum().sort_values(ascending=False)
 percent =(data.isnull().sum() /data.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(20 )<data_type_conversions>
labels = pd.DataFrame(glob.glob('.. /input/annotation/Annotation/**/**'), columns=['Path'])
 labels['Labels'] = labels['Path'].map(lambda x: x.split('/')[4].split('-')[1])
 labels['FileName'] = labels['Path'].map(lambda x: x.split('/')[-1] + '.jpg')
 labels = {f:l for f,l in labels[['FileName', 'Labels']].values}
dog_bytes['FileName'] = dog_bytes['Path'].map(lambda x: x.split('/')[-1])
 dog_bytes['Labels'] = dog_bytes['FileName'].map(labels)
 dog_bytes.head()
<count_missing_values>
dog_bytes['Group'] = cluster.KMeans(n_clusters=400, random_state=4, n_jobs=-1 ).fit_predict(dog_bytes[list(range(30)) ])
 dog_bytes['Group'].value_counts() [:5]
print([column for column in data1.columns if data1[column].isnull().sum() >0])
<split>
fig = plt.figure(figsize=(8, 80))
samples = []
for i in range(400):
g = dog_bytes[dog_bytes['Group'] == i]
if len(g)> 5:
samples += list(g['Path'].values[:5])
 for i in range(len(samples)) [:50]:
ax = fig.add_subplot(len(samples)/5, 5, i+1, xticks=[], yticks=[])
 img = Image.open(samples[i])
 img = img.resize(( 100,int(img.size[1]/(img.size[0]/100))), Image.ANTIALIAS)
 img = img.crop(( 0, 0, 64, 64))
plt.imshow(img )
<prepare_x_and_y><EOS> 
def sim_img(path):
img = Image.open(path ).convert('RGB')
 img = img.resize(( 100,int(img.size[1]/(img.size[0]/100))), Image.ANTIALIAS)
 img = img.crop(( 0, 0, 64, 64))
return img
samples = []
for i in range(400):
g = dog_bytes[dog_bytes['Group'] == i]
p = g['Path'].values
for i in range(0,len(p)-2, 2):
samples.append([p[i],p[i+1]])
 if len(samples)> 11_000: break
for i in range(0,len(samples)-1):
samples.append([samples[i][0],samples[i+1][0]])
 if len(samples)> 11_000: break
print(len(samples))