Kaggle competition: Dog Breed Identification
<SOS> metric: MulticlassLoss Kaggle data source: dog-breed-identification<split>
import numpy as np
import pandas as pd
import cv2
import seaborn as sns
import matplotlib.pyplot as plt
from IPython.display import clear_output
from keras import backend as K
from keras.layers import Dense, Activation, Dropout, BatchNormalization, Input, Flatten, Conv2D, MaxPooling2D, Lambda, UpSampling2D, Concatenate
from keras.models import Model
from keras.optimizers import Adam
from keras.callbacks import Callback, EarlyStopping, ReduceLROnPlateau
from keras.applications.inception_resnet_v2 import InceptionResNetV2
from keras.initializers import he_normal
from keras.preprocessing.image import ImageDataGenerator
import os
import shutil
X_train, X_test, y_train, y_test = train_test_split(new_training_data, target, test_size = 0.20, random_state = 97)
<define_variables>
labels = pd.read_csv('.. /input/dog-breed-identification/labels.csv')
 labels.head(5 )
start = timeit.default_timer()
numberOfParents = 8
numberOfParentsMating = 4
numberOfParameters = 5
numberOfGenerations = 4
populationSize =(numberOfParents, numberOfParameters)
 population = initilialize_poplulation(numberOfParents)
 fitnessHistory = np.empty([numberOfGenerations+1, numberOfParents])
 populationHistory = np.empty([(numberOfGenerations+1)*numberOfParents, numberOfParameters])
 populationHistory[0:numberOfParents, :] = population
for generation in range(numberOfGenerations):
print("This is number %s generation" %(generation))
fitnessValue = train_population(population=population, train=X_train,y_train=y_train, test=X_test, y_test=y_test)
 fitnessHistory[generation, :] = fitnessValue
print('Best accuracy score in the this iteration = {}'.format(np.max(fitnessHistory[generation, :])))
 parents = new_parents_selection(population=population, fitness=fitnessValue, numParents=numberOfParentsMating)
 children = crossover_uniform(parents=parents, childrenSize=(populationSize[0] - parents.shape[0], numberOfParameters))
children_mutated = mutation(children, numberOfParameters)

population[0:parents.shape[0], :] = parents
population[parents.shape[0]:, :] = children_mutated
populationHistory[(generation+1)*numberOfParents :(generation+1)*numberOfParents+ numberOfParents , :] = population
stop = timeit.default_timer()
print('Execution time: ',(stop - start)/60 )<train_model>
classes = np.unique(labels.breed)
 classes_num = classes.size
classes_num
fitness = train_population(population=population, train=X_train,y_train=y_train, test=X_test, y_test=y_test)
 fitnessHistory[generation+1, :] = fitness
bestFitnessIndex = np.where(fitness == np.max(fitness)) [0][0]
print("Best fitness is =", fitness[bestFitnessIndex])
 print("Best parameters are:")
 print('Max_depth', int(population[bestFitnessIndex][0]))
print('n_estimators', int(population[bestFitnessIndex][1]))
print('min_samples_split', int(population[bestFitnessIndex][2]))
print('min_samples_leaf', int(population[bestFitnessIndex][3]))
print('bootstrap', int(population[bestFitnessIndex][4])) 
<save_to_csv>
train_dir = '.. /input/dog-breed-identification/train'
images_names = os.listdir(train_dir)
 images_num = len(images_names)
 print(f'Number of images: {images_num}' )
model=RandomForestClassifier(n_estimators=int(population[bestFitnessIndex][1]),max_depth=int(population[bestFitnessIndex][0]),
min_samples_leaf=int(population[bestFitnessIndex][3]),min_samples_split=int(population[bestFitnessIndex][2]))
model.fit(new_training_data,target)
 y_pred = model.predict(new_test_data)
 sub['surface'] = le.inverse_transform(y_pred)
 sub.to_csv('random_f_ga.csv', index=False)
<set_options>
labels_jpg = labels.copy(deep=True)
 labels_jpg['id'] += '.jpg'
grouped_ids = labels_jpg.groupby('breed')['id'].apply(list ).to_dict()
print(classes[0], grouped_ids[classes[0]] )
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
 pd.set_option('max_columns', None)
<load_from_csv>
test_split = 0.1
valid_split = 0.2
%%time
X_train = pd.read_csv(os.path.join(location, 'X_train.csv'))
X_test = pd.read_csv(os.path.join(location, 'X_test.csv'))
y_train = pd.read_csv(os.path.join(location, 'y_train.csv'))<train_model>
train_size = 0
valid_size = 0
test_size = 0
for breed_idx,(breed, breed_images)in enumerate(grouped_ids.items()):
for img in breed_images:
rnd_prob = np.random.rand()
if rnd_prob <= test_split:
shutil.copy(train_dir+'/'+img, new_test_dir+'/'+breed)
 test_size += 1
elif rnd_prob <=(test_split + valid_split):
shutil.copy(train_dir+'/'+img, new_valid_dir+'/'+breed)
 valid_size += 1
else:
shutil.copy(train_dir+'/'+img, new_train_dir+'/'+breed)
 train_size += 1
clear_output(wait=True)
 print(f'Organized {breed_idx+1} out of {classes_num} breeds: {breed}' )
print("Xtrain: {}
Xtest: {}
ytrain: {}".format(X_train.shape, X_test.shape, y_train.shape))<train_model>
norm_factor = 1 / 255
transform_params = {
'featurewise_center': False,
'featurewise_std_normalization': False,
'samplewise_center': False,
'samplewise_std_normalization': False,
'rotation_range': 30,
'width_shift_range': 0.15,
'height_shift_range': 0.15,
'horizontal_flip': True,
'rescale': norm_factor
}
img_gen = ImageDataGenerator(**transform_params )
print('Size of the Xtrain')
 print('Numbers of Measurements: {0}
Numbers of columns: {1}'.format(X_train.shape[0], X_train.shape[1]))<train_model>
img_feed = ImageDataGenerator(rescale=1/255 )
print('Size of the Xtest')
 print('Numbers of Measurements: {0}
Numbers of columns: {1}'.format(X_test.shape[0], X_test.shape[1]))<train_model>
plateau_reduce = ReduceLROnPlateau(monitor='val_loss', factor=0.01,
patience=1, min_lr=1e-20 )
print('Size of the Labels')
 print('Numbers of Measurements: {0}
Numbers of columns: {1}'.format(y_train.shape[0], y_train.shape[1]))<feature_engineering>
e_stop = EarlyStopping(monitor='val_loss', patience=15, mode='min', restore_best_weights=True )
def missing_data(data):
total = data.isnull().sum()
percent =(total/data.isnull().count() *100)
 miss_column = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 miss_column['Types'] = types
return(np.transpose(miss_column))<count_missing_values>
callbacks = [plotter, plateau_reduce, e_stop]
missing_data(X_train )<count_missing_values>
def dense_block(x, neurons, layer_no):
x = Dense(neurons, kernel_initializer=he_normal(layer_no), name=f'topDense{layer_no}' )(x)
 x = Activation('relu', name=f'Relu{layer_no}' )(x)
 x = BatchNormalization(name=f'BatchNorm{layer_no}' )(x)
 x = Dropout(0.5, name=f'Dropout{layer_no}' )(x)
 return x
missing_data(X_test )<count_missing_values>
def create_model(shape):
input_layer = Input(shape, name='input_layer')
 incep_res = InceptionResNetV2(include_top=False, weights='imagenet', input_tensor=input_layer)
 for layer in incep_res.layers:
layer.trainable = False
pool = MaxPooling2D(pool_size=[3, 3], strides=[3, 3], padding='same' )(incep_res.output)
 flat1 = Flatten(name='Flatten1' )(pool)
 flat1_bn = BatchNormalization(name='BatchNormFlat' )(flat1)
 dens1 = dense_block(flat1_bn, neurons=512, layer_no=1)
 dens2 = dense_block(dens1, neurons=512, layer_no=2)
 dens3 = dense_block(dens2, neurons=1024, layer_no=3)
 dens_final = Dense(classes_num, name='Dense4' )(dens3)
 output_layer = Activation('softmax', name='Softmax' )(dens_final)
 model = Model(inputs=[input_layer], outputs=[output_layer])
 return model
missing_data(y_train )<count_values>
height, width, channels_num = 512, 512, 3
learning_rate = 0.004
epochs = 15
batch_size = 32
Surface_count = y_train['surface'].value_counts().reset_index().rename(columns = {'index' : 'Labels'})
 Surface_count<normalization>
model = create_model(( height, width, channels_num))
optimizer = Adam(learning_rate)
 model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['acc'])
 model.summary()
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z
def data_engineering(actual):
new = pd.DataFrame()
actual['total_angular_velocity'] =(actual['angular_velocity_X'] ** 2 + actual['angular_velocity_Y'] ** 2 + actual['angular_velocity_Z'] ** 2)** 0.5
actual['total_linear_acceleration'] =(actual['linear_acceleration_X'] ** 2 + actual['linear_acceleration_Y'] ** 2 + actual['linear_acceleration_Z'] ** 2)** 0.5
actual['acc_vs_vel'] = actual['total_linear_acceleration'] / actual['total_angular_velocity']
x, y, z, w = actual['orientation_X'].tolist() , actual['orientation_Y'].tolist() , actual['orientation_Z'].tolist() , actual['orientation_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 actual['euler_x'] = nx
actual['euler_y'] = ny
actual['euler_z'] = nz
actual['total_angle'] =(actual['euler_x'] ** 2 + actual['euler_y'] ** 2 + actual['euler_z'] ** 2)** 5
actual['angle_vs_acc'] = actual['total_angle'] / actual['total_linear_acceleration']
actual['angle_vs_vel'] = actual['total_angle'] / actual['total_angular_velocity']
def f1(x):
return np.mean(np.diff(np.abs(np.diff(x))))
def f2(x):
return np.mean(np.abs(np.diff(x)))
 for col in actual.columns:
if col in ['row_id', 'series_id', 'measurement_number']:
continue
new[col + '_mean'] = actual.groupby(['series_id'])[col].mean()
new[col + '_min'] = actual.groupby(['series_id'])[col].min()
new[col + '_max'] = actual.groupby(['series_id'])[col].max()
new[col + '_std'] = actual.groupby(['series_id'])[col].std()
new[col + '_max_to_min'] = new[col + '_max'] / new[col + '_min']
new[col + '_mean_abs_change'] = actual.groupby('series_id')[col].apply(f2)
 new[col + '_mean_change_of_abs_change'] = actual.groupby('series_id')[col].apply(f1)
 new[col + '_abs_max'] = actual.groupby('series_id')[col].apply(lambda x: np.max(np.abs(x)))
 new[col + '_abs_min'] = actual.groupby('series_id')[col].apply(lambda x: np.min(np.abs(x)))
 return new<compute_test_metric>
train_gen = img_gen.flow_from_directory(directory=new_train_dir, target_size=(height, width), color_mode='rgb', classes=list(classes),
class_mode='categorical', batch_size=batch_size, shuffle=True, interpolation='nearest')
 valid_gen = img_feed.flow_from_directory(directory=new_valid_dir, target_size=(height, width), color_mode='rgb', classes=list(classes),
class_mode='categorical', batch_size=batch_size, shuffle=True, interpolation='nearest')
%%time
xtrain = data_engineering(X_train)
 xtest = data_engineering(X_test )<categorify>
!wget "https://drive.google.com/uc?id=1-3Y-DB5uhOaY69pvVl5rmfirw9aKCYYB&export=download" -O '/root/training_curves.npy'
le = LabelEncoder()
y_train['surface'] = le.fit_transform(y_train['surface'] )<categorify>
!wget "https://drive.google.com/uc?id=1Omp4wFOWc2WslToduWAgBkyRv9poQVKb&export=download" -O '/root/decoder_weights.npy'
xtrain.fillna(0, inplace = True)
 xtrain.replace(-np.inf, 0, inplace = True)
 xtrain.replace(np.inf, 0, inplace = True)
 xtest.fillna(0, inplace = True)
 xtest.replace(-np.inf, 0, inplace = True)
 xtest.replace(np.inf, 0, inplace = True )<split>
decoder_weights = np.load('/root/decoder_weights.npy', allow_pickle=True ).item()
for layer_name, layer_weights in decoder_weights.items() :
model.get_layer(layer_name ).set_weights(layer_weights )
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=11)
 def feed_model(train,test,label,folds=folds,averaging='usual',clf=None,clf_type='rfc',params=None,
plot_feature_importance=False,groups=y_train['group_id']):
forcast = np.zeros(( test.shape[0], 9))
con_pred = np.zeros(( train.shape[0]))
scores = []
feature_importance = pd.DataFrame()
for fold_,(train_index, valid_index)in enumerate(folds.split(train, label, groups)) :
print('Fold', fold_, 'started at', time.ctime())
 x_train, x_valid = train.iloc[train_index], train.iloc[valid_index]
y_train, y_valid = label.iloc[train_index], label.iloc[valid_index]
if clf_type == 'rfc':
clf = clf
clf.fit(x_train, y_train)
 Valid_pred = clf.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, Valid_pred)
 Real_pred = clf.predict_proba(test)
 con_pred[valid_index] = clf.predict(x_valid ).reshape(-1,)
 scores.append(accuracy_score(y_valid, Valid_pred))
if averaging == 'usual':
forcast += Real_pred
elif averaging == 'rank':
forcast += pd.Series(Real_pred ).rank().values
forcast /= folds.n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if clf_type == 'lgb':
feature_importance["importance"] /= folds.n_splits
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(by="importance",
ascending=False)[:50].index
best_features = feature_importance[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return Valid_pred, forcast, feature_importance
return Valid_pred, forcast, scores
else:
return Valid_pred, forcast, scores 
<normalization>
metrics = model.evaluate_generator(test_flow, steps=test_size )
sfm = SelectFromModel(clf, threshold=0.001)
 sfm.fit(xtrain, y_train['surface'])
 for feature_list_index in sfm.get_support(indices=True):
print(feat_labels[feature_list_index])
 xtrain_importance = sfm.transform(xtrain)
 xtest_importance = sfm.transform(xtest )<split>
m_names = model.metrics_names
print(f'{m_names[0]} = {metrics[0]}
{m_names[1]} = {metrics[1]}' )
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=11)
 def feed_model_importance(train,test,label,folds=folds,averaging='usual',clf=None,clf_type='rfc',params=None,
plot_feature_importance=False,groups=y_train['group_id']):
forcast = np.zeros(( test.shape[0], 9))
con_pred = np.zeros(( train.shape[0]))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(train, label, groups)) :
print('Fold', fold_n, 'started at', time.ctime())
 x_train, x_valid = train[train_index], train[valid_index]
y_train, y_valid = label[train_index], label[valid_index]
if clf_type == 'rfc':
clf = clf
clf.fit(x_train, y_train)
 Valid_pred = clf.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, Valid_pred)
 Real_pred = clf.predict_proba(test)
 con_pred[valid_index] +=(Valid_pred ).reshape(-1,)
 scores.append(accuracy_score(y_valid, Valid_pred))
if averaging == 'usual':
forcast += Real_pred
elif averaging == 'rank':
forcast += pd.Series(Real_pred ).rank().values
forcast /= folds.n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if clf_type == 'lgb':
feature_importance["importance"] /= n_folds
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return Valid_pred, forcast, feature_importance
return Valid_pred, forcast, scores
else:
return Valid_pred, forcast, scores 
<choose_model_class>
one_hot_map = train_gen.class_indices
one_hot_map
np.random.seed(0)
 folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=11)
 def create_network() :
network = models.Sequential()
network.add(layers.Dense(units=16, activation='relu', input_dim= 171))
network.add(layers.Dense(units=16, activation='relu'))
network.add(layers.Dense(units=1, activation='sigmoid'))
network.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return network
es = EarlyStopping(monitor='val_loss',mode='auto',verbose=1,patience=20)
 mc = ModelCheckpoint('best_model.h5',monitor='val_acc',mode='max',verbose=1,save_best_only=True)
 neural_network = KerasClassifier(build_fn=create_network,epochs=10,batch_size=100,verbose=0)
 cross_val_score(neural_network, xtrain, y_train['surface'],fit_params={'callbacks': [es,mc]}, cv=folds )<save_to_csv>
input_dir = '.. /input/dog-breed-identification'
sub = pd.read_csv(os.path.join(location,'sample_submission.csv'))
sub['surface'] = le.inverse_transform(forcast_rfc.argmax(axis=1))
sub.to_csv('Forcasting.csv', index=False)
 sub.head(20 )<set_options>
test_names = os.listdir(input_dir+'/test')
 test_names.sort()
test_size = len(test_names)
 test_size
warnings.filterwarnings('ignore' )<load_from_csv>
<drop_column><EOS> 
!wget 'https://drive.google.com/uc?id=18IQ9WVf1KcVTgwTgGZXKzx61W0ISayI8&export=download' -O 'submission_colab.csv'