Kaggle competition: Aerial Cactus Identification
def generate_data(x1,x2,df,t):
X = pd.concat([df[x1], df[x2]], axis=1 ).values
if t==1:
y = train_y.values.astype(int)
 return(X, y)
 else:
return X<define_variables>
%%time
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/sample_submission.csv')
 df_train.head(3 )
datasets = []
for i in range(len(clustered_features)) :
datasets.append(( generate_data(feature_first,clustered_features[i],train_x_all,1),{}))<create_dataframe>
TRAIN_DATA_PATH ='.. /input/train/train'
sample = TRAIN_DATA_PATH +'/'+ os.listdir(TRAIN_DATA_PATH)[0]
TEST_DATA_PATH = '.. /input/test/test'
np.array(Image.open(sample)).shape
datasets_test = []
rez = pd.DataFrame(index = test_x.index)
 for i in range(len(clustered_features)) :
datasets_test.append(generate_data(feature_first,clustered_features[i],test_x_all,0))<define_search_model>
print('target 0:1->',len(df_train[df_train.has_cactus==0])/len(df_train[df_train.has_cactus==1]))
def generate_clustering_algorithms(Z,n_clusters):
bandwidth = cluster.estimate_bandwidth(df, quantile=params['quantile'])
 connectivity = kneighbors_graph(
 Z, n_neighbors=params['n_neighbors'], include_self=False)
 connectivity = 0.5 *(connectivity + connectivity.T)
 ms = cluster.MeanShift(bandwidth=bandwidth, bin_seeding=True)
 kmeans = cluster.KMeans(n_clusters=n_clusters, random_state = 1000)
 two_means = cluster.MiniBatchKMeans(n_clusters=n_clusters)
 ward = cluster.AgglomerativeClustering(n_clusters=n_clusters, linkage='ward',
connectivity=connectivity)
 spectral = cluster.SpectralClustering(n_clusters=n_clusters, eigen_solver='arpack',
affinity="nearest_neighbors")
 dbscan = cluster.DBSCAN(eps=params['eps'])
 optics = cluster.OPTICS(min_samples=params['min_samples'],
xi=params['xi'],
min_cluster_size=params['min_cluster_size'])
 affinity_propagation = cluster.AffinityPropagation(damping=params['damping'])
 average_linkage = cluster.AgglomerativeClustering(linkage="average", affinity="cityblock",
n_clusters=params['n_clusters'], connectivity=connectivity)
 birch = cluster.Birch(n_clusters=params['n_clusters'])
 gmm = mixture.GaussianMixture(n_components=n_clusters, covariance_type='full')
 clustering_algorithms =(
('MiniBatchKMeans', two_means), 
('KMeans', kmeans), 
('AffinityPropagation', affinity_propagation), 
('MeanShift', ms), 
('SpectralClustering', spectral), 
('Ward', ward), 
('AgglomerativeClustering', average_linkage), 
('DBSCAN', dbscan), 
('OPTICS', optics), 
('Birch', birch), 
('GaussianMixture', gmm)
)
 return clustering_algorithms<concatenate>
%%time
tmp = []
for i in range(len(df_train)) :
tmp.append(np.array(Image.open(TRAIN_DATA_PATH +'/'+ df_train.iloc[i]['id'])))
 X_train, y_train = np.array(tmp)/255, df_train['has_cactus']
tmp = []
for i in range(len(df_test)) :
tmp.append(np.array(Image.open(TEST_DATA_PATH +'/'+ df_test.iloc[i]['id'])))
 X_test = np.array(tmp)/255
del tmp
print(X_train.shape, y_train.shape)
 print(X_test.shape )
train_x = pd.concat([train_x_all.WomanOrBoySurvived.fillna(0),
train_x_all.Alone,
train_x_all.Sex,
], axis=1)
 test_x = pd.concat([test_x_all.WomanOrBoySurvived.fillna(0),
test_x_all.Alone,
test_x_all.Sex,
], axis=1 )<merge>
def build_model(input_shape):
model =Sequential()
model.add(Conv2D(64,(3,3),padding='same',input_shape=(input_shape)))
 model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(Conv2D(64,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(Conv2D(64,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(MaxPooling2D())
 model.add(Dropout(0.5))
model.add(Conv2D(128,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(Conv2D(128,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(Conv2D(128,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(MaxPooling2D())
 model.add(Dropout(0.5))
model.add(Conv2D(256,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(Conv2D(256,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(Conv2D(256,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(BatchNormalization(scale=False))
model.add(GlobalAveragePooling2D())
 model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile('adam',loss='binary_crossentropy',metrics=['accuracy'])
 return model
train_x = train_x.join(train_x_all[rez_col], how='left', lsuffix="_rez")
 train_x.head(2 )<merge>
1.0 / len(df_train[df_train.has_cactus==0])* len(df_train[df_train.has_cactus==1] )
test_x = test_x.join(rez[rez_col], how='left', lsuffix="_rez")
 test_x.head(2 )<train_on_grid>
%%time
histories = []
oof_pred = np.zeros(len(df_train))
sub_pred = np.zeros(len(df_test))
class_weights = {}
weights = [3.010082493125573,
1.0]
len(df_train[df_train.has_cactus==0])
 for i in range(2):
class_weights[i] = weights[i]
print('class_weights:',class_weights)
 checkpoint_name = '/checkpoint.file'
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)
 BATCH_SIZE = 64
EPOCHS = 128
for fold_id,(train_index, val_index)in enumerate(skf.split(X_train, y_train)) :
print(f'fold id: {fold_id}')
 X_tr, y_tr = X_train[train_index], y_train[train_index]
X_val, y_val = X_train[val_index], y_train[val_index]
callbacks=[
keras.callbacks.ModelCheckpoint(
 checkpoint_name, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False),
keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.7, patience=5, verbose=1,min_delta=0.00005,),
keras.callbacks.EarlyStopping(monitor='val_loss', patience=15)
 ]
datagen = ImageDataGenerator(
 height_shift_range=0.1,
horizontal_flip=True 
)
 K.clear_session()
model = build_model(X_train.shape[1:])
 model.summary()
histories.append(
 model.fit_generator(
 datagen.flow(X_train, y_train, batch_size=BATCH_SIZE),
steps_per_epoch=int(np.ceil(len(X_train)/ BATCH_SIZE)) , validation_data=(X_val, y_val),
epochs=EPOCHS, class_weight=class_weights,
callbacks=callbacks, verbose=2)
)
 model = load_model(checkpoint_name)
 oof_pred[val_index] = model.predict(X_val ).flatten()
sub_pred += model.predict(X_test ).flatten() / skf.n_splits
print(roc_auc_score(y_val, oof_pred[val_index]))
plot_roc_auc(y_val, oof_pred[val_index])
 del callbacks
sub_pred = np.clip(sub_pred,0.0,1.0 )
parameters = {'max_depth' : np.arange(2, 9, dtype=int),
'min_samples_leaf' : np.arange(1, 4, dtype=int)}
classifier = DecisionTreeClassifier(random_state=1000)
 model = GridSearchCV(estimator=classifier, param_grid=parameters, scoring='accuracy', cv=10, n_jobs=-1)
 model.fit(train_x, train_y)
 best_parameters = model.best_params_
print(best_parameters )<train_model>
print('Ambiguous image index:',np.where(( sub_pred<0.80)&(sub_pred>0.20)) [:32][0] )
model=DecisionTreeClassifier(max_depth = best_parameters['max_depth'],
random_state = 1118)
 model.fit(train_x, train_y )<save_to_csv>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['has_cactus'] = sub_pred
submission.to_csv('submission.csv', index=False)
 submission.head(50 )