Kaggle competition: Statoil/C-CORE Iceberg Classifier Challenge
<SOS> metric: LogLoss Kaggle data source: statoil-iceberg-classifier-challenge<feature_engineering>
np.random.seed(98643)
 tf.set_random_seed(683)
 denoise_wavelet, estimate_sigma, denoise_tv_bregman, denoise_nl_means)
 %matplotlib inline
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1 )<feature_engineering>
train = pd.read_json(".. /input/train.json")
 train.inc_angle = train.inc_angle.replace('na', 0)
 train.inc_angle = train.inc_angle.astype(float ).fillna(0.0)
 train_all = True
train_b = True or train_all
train_img = True or train_all
train_total = True or train_all
predict_submission = True and train_all
clean_all = False
clean_b = False or clean_all
clean_img = False or clean_all
load_all = False
load_b = False or load_all
load_img = False or load_all
train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))<feature_engineering>
def create_dataset(frame, labeled, smooth_rgb=0.2, smooth_gray=0.5,
weight_rgb=0.05, weight_gray=0.05):
band_1, band_2, images = frame['band_1'].values, frame['band_2'].values, color_composite(frame)
 to_arr = lambda x: np.asarray([np.asarray(item)for item in x])
 band_1 = to_arr(band_1)
 band_2 = to_arr(band_2)
 band_3 =(band_1 + band_2)/ 2
gray_reshape = lambda x: np.asarray([item.reshape(75, 75)for item in x])
 band_1 = gray_reshape(band_1)
 band_2 = gray_reshape(band_2)
 band_3 = gray_reshape(band_3)
 print('Denoising and reshaping')
 if train_b and clean_b:
band_1 = smooth(denoise(band_1, weight_gray, False), smooth_gray)
 print('Gray 1 done')
 band_2 = smooth(denoise(band_2, weight_gray, False), smooth_gray)
 print('Gray 2 done')
 band_3 = smooth(denoise(band_3, weight_gray, False), smooth_gray)
 print('Gray 3 done')
 if train_img and clean_img:
images = smooth(denoise(images, weight_rgb, True), smooth_rgb)
 print('RGB done')
 tf_reshape = lambda x: np.asarray([item.reshape(75, 75, 1)for item in x])
 band_1 = tf_reshape(band_1)
 band_2 = tf_reshape(band_2)
 band_3 = tf_reshape(band_3)
 band = np.concatenate([band_1, band_2, band_3], axis=3)
 if labeled:
y = np.array(frame["is_iceberg"])
 else:
y = None
return y, band, images
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1 )<drop_column>
y_train, X_b, X_images = create_dataset(train, True )
train = train.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate'], axis=1 )<count_values>
def get_model_notebook(lr, decay, channels, relu_type='relu'):
input_1 = Input(shape=(75, 75, channels))
fcnn = Conv2D(32, kernel_size=(3, 3), activation=relu_type )(
 BatchNormalization()(input_1))
fcnn = MaxPooling2D(( 3, 3))(fcnn)
 fcnn = Dropout(0.2 )(fcnn)
 fcnn = Conv2D(64, kernel_size=(3, 3), activation=relu_type )(fcnn)
 fcnn = MaxPooling2D(( 2, 2), strides=(2, 2))(fcnn)
 fcnn = Dropout(0.2 )(fcnn)
 fcnn = Conv2D(128, kernel_size=(3, 3), activation=relu_type )(fcnn)
 fcnn = MaxPooling2D(( 2, 2), strides=(2, 2))(fcnn)
 fcnn = Dropout(0.2 )(fcnn)
 fcnn = Conv2D(128, kernel_size=(3, 3), activation=relu_type )(fcnn)
 fcnn = MaxPooling2D(( 2, 2), strides=(2, 2))(fcnn)
 fcnn = Dropout(0.2 )(fcnn)
 fcnn = BatchNormalization()(fcnn)
 fcnn = Flatten()(fcnn)
 local_input = input_1
partial_model = Model(input_1, fcnn)
 dense = Dropout(0.2 )(fcnn)
 dense = Dense(256, activation=relu_type )(dense)
 dense = Dropout(0.2 )(dense)
 dense = Dense(128, activation=relu_type )(dense)
 dense = Dropout(0.2 )(dense)
 dense = Dense(64, activation=relu_type )(dense)
 dense = Dropout(0.2 )(dense)
 output = Dense(1, activation="sigmoid" )(dense)
 model = Model(local_input, output)
 optimizer = Adam(lr=lr, decay=decay)
 model.compile(loss="binary_crossentropy", optimizer=optimizer, metrics=["accuracy"])
 return model, partial_model
train['WindSpeed'].value_counts()<feature_engineering>
def combined_model(m_b, m_img, lr, decay):
input_b = Input(shape=(75, 75, 3))
input_img = Input(shape=(75, 75, 3))
m1 = m_b(input_b)
 m2 = m_img(input_img)
 common = Concatenate()([m1, m2])
 common = BatchNormalization()(common)
 common = Dropout(0.3 )(common)
 common = Dense(1024, activation='relu' )(common)
 common = Dropout(0.3 )(common)
 common = Dense(512, activation='relu' )(common)
 common = Dropout(0.3 )(common)
 output = Dense(1, activation="sigmoid" )(common)
 model = Model([input_b, input_img], output)
 optimizer = Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=decay)
 model.compile(loss="binary_crossentropy", optimizer=optimizer, metrics=["accuracy"])
 return model
train['WindSpeed'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x )<count_values>
def gen_flow_multi_inputs(I1, I2, y, batch_size):
gen1 = ImageDataGenerator(horizontal_flip=True,
vertical_flip=True,
width_shift_range=0.,
height_shift_range=0.,
channel_shift_range=0,
zoom_range=0.2,
rotation_range=10)
 gen2 = ImageDataGenerator(horizontal_flip=True,
vertical_flip=True,
width_shift_range=0.,
height_shift_range=0.,
channel_shift_range=0,
zoom_range=0.2,
rotation_range=10)
 genI1 = gen1.flow(I1, y, batch_size=batch_size, seed=57, shuffle=False)
 genI2 = gen2.flow(I1, I2, batch_size=batch_size, seed=57, shuffle=False)
 while True:
I1i = genI1.next()
I2i = genI2.next()
np.testing.assert_array_equal(I2i[0], I1i[0])
 yield [I1i[0], I2i[1]], I1i[1]
train['WindSpeed'].value_counts()<feature_engineering>
def train_model(model, batch_size, epochs, checkpoint_name, X_train, y_train, val_data, verbose=2):
callbacks = [ModelCheckpoint(checkpoint_name, save_best_only=True, monitor='val_loss')]
datagen = ImageDataGenerator(horizontal_flip=True,
vertical_flip=True,
width_shift_range=0.,
height_shift_range=0.,
channel_shift_range=0,
zoom_range=0.2,
rotation_range=10)
 x_test, y_test = val_data
try:
model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size), epochs=epochs,
steps_per_epoch=len(X_train)/ batch_size,
validation_data=(x_test, y_test), verbose=1,
callbacks=callbacks)
 except KeyboardInterrupt:
if verbose > 0:
print('Interrupted')
 if verbose > 0:
print('Loading model')
 model.load_weights(filepath=checkpoint_name)
 return model
train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x )<data_type_conversions>
def gen_model_weights(lr, decay, channels, relu, batch_size, epochs, path_name, data, verbose=2):
X_train, y_train, X_test, y_test, X_val, y_val = data
model, partial_model = get_model_notebook(lr, decay, channels, relu)
 model = train_model(model, batch_size, epochs, path_name,
X_train, y_train,(X_test, y_test), verbose=verbose)
 if verbose > 0:
loss_val, acc_val = model.evaluate(X_val, y_val,
verbose=0, batch_size=batch_size)
 loss_train, acc_train = model.evaluate(X_test, y_test,
verbose=0, batch_size=batch_size)
 print('Val/Train Loss:', str(loss_val)+ '/' + str(loss_train), \
'Val/Train Acc:', str(acc_val)+ '/' + str(acc_train))
return model, partial_model
def str_to_float(txt):
try:
return float(txt)
 except:
return -1<feature_engineering>
def train_models(dataset, lr, batch_size, max_epoch, verbose=2, return_model=False):
y_train, X_b, X_images = dataset
y_train_full, y_val,\
X_b_full, X_b_val,\
X_images_full, X_images_val = train_test_split(y_train, X_b, X_images, random_state=687, train_size=0.9)
 y_train, y_test, \
X_b_train, X_b_test, \
X_images_train, X_images_test = train_test_split(y_train_full, X_b_full, X_images_full, random_state=576, train_size=0.85)
 if train_b:
if verbose > 0:
print('Training bandwidth network')
 data_b1 =(X_b_train, y_train, X_b_test, y_test, X_b_val, y_val)
 model_b, model_b_cut = gen_model_weights(lr, 1e-6, 3, 'relu', batch_size, max_epoch, 'model_b',
data=data_b1, verbose=verbose)
 if train_img:
if verbose > 0:
print('Training image network')
 data_images =(X_images_train, y_train, X_images_test, y_test, X_images_val, y_val)
 model_images, model_images_cut = gen_model_weights(lr, 1e-6, 3, 'relu', batch_size, max_epoch, 'model_img',
data_images, verbose=verbose)
 if train_total:
common_model = combined_model(model_b_cut, model_images_cut, lr/2, 1e-7)
 common_x_train = [X_b_full, X_images_full]
common_y_train = y_train_full
common_x_val = [X_b_val, X_images_val]
common_y_val = y_val
if verbose > 0:
print('Training common network')
 callbacks = [ModelCheckpoint('common', save_best_only=True, monitor='val_loss')]
try:
common_model.fit_generator(gen_flow_multi_inputs(X_b_full, X_images_full, y_train_full, batch_size),
epochs=30,
steps_per_epoch=len(X_b_full)/ batch_size,
validation_data=(common_x_val, common_y_val), verbose=1,
callbacks=callbacks)
 except KeyboardInterrupt:
pass
common_model.load_weights(filepath='common')
 loss_val, acc_val = common_model.evaluate(common_x_val, common_y_val,
verbose=0, batch_size=batch_size)
 loss_train, acc_train = common_model.evaluate(common_x_train, common_y_train,
verbose=0, batch_size=batch_size)
 if verbose > 0:
print('Loss:', loss_val, 'Acc:', acc_val)
 if return_model:
return common_model
else:
return(loss_train, acc_train),(loss_val, acc_val )
train['WindSpeed'] = train['WindSpeed'].apply(str_to_float )<count_values>
common_model = train_models(( y_train, X_b, X_images), 7e-04, 32, 50, 1, return_model=True )
<drop_column><EOS> 
if predict_submission:
print('Reading test dataset')
 test = pd.read_json(".. /input/test.json")
 y_fin, X_fin_b, X_fin_img = create_dataset(test, False)
 print('Predicting')
 prediction = common_model.predict([X_fin_b, X_fin_img], verbose=1, batch_size=32)
 print('Submitting')
 submission = pd.DataFrame({'id': test["id"], 'is_iceberg': prediction.reshape(( prediction.shape[0])) })
 submission.to_csv("./submission.csv", index=False)
 print('Done' )