Kaggle competition: Statoil/C-CORE Iceberg Classifier Challenge
<SOS> metric: LogLoss Kaggle data source: statoil-iceberg-classifier-challenge<load_from_csv>
%matplotlib inline
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train rows and columns : ", train_df.shape)
 print("Test rows and columns : ", test_df.shape )<sort_values>
train = pd.read_json(".. /input/train.json" )
missing_df = train_df.isnull().sum(axis=0 ).reset_index()
missing_df.columns = ['column_name', 'missing_count']
missing_df = missing_df[missing_df['missing_count']>0]
missing_df = missing_df.sort_values(by='missing_count')
 missing_df<groupby>
def standardise_data(data):

 out =(np.array(data)- np.mean(data)) / np.std(data)
 return out.tolist()
def process_data(data, predict=False):
data["band_1"] = data["band_1"].apply(standardise_data)
 data["band_2"] = data["band_2"].apply(standardise_data)
 band1 = np.array([np.array(band ).astype(np.float32 ).reshape(75,75)for band in data["band_1"]])
 band2 = np.array([np.array(band ).astype(np.float32 ).reshape(75,75)for band in data["band_2"]])
 X = np.concatenate([band1[:, :, :, np.newaxis],
band2[:, :, :, np.newaxis], 
(( band1+band2)/2)[:, :, :, np.newaxis]],
axis=-1)
 if predict==False:
y = np.array(data['is_iceberg'])
 return X, y
return X
dtype_df = train_df.dtypes.reset_index()
dtype_df.columns = ["Count", "Column Type"]
dtype_df.groupby("Column Type" ).aggregate('count' ).reset_index()<count_unique_values>
X_train, y_train = process_data(train )
unique_df = train_df.nunique().reset_index()
unique_df.columns = ["col_name", "unique_count"]
constant_df = unique_df[unique_df["unique_count"]==1]
constant_df.shape<prepare_x_and_y>
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Input, Flatten, Activation, BatchNormalization
from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint, Callback, EarlyStopping
train_X = train_df.drop(constant_df.col_name.tolist() + ["ID", "target"], axis=1)
 test_X = test_df.drop(constant_df.col_name.tolist() + ["ID"], axis=1)
 train_y = np.log1p(train_df["target"].values )<create_dataframe>
def getModel() :
drop = 0.2
gmodel=Sequential()
gmodel.add(Conv2D(32, kernel_size=(3, 3),activation='relu', input_shape=(75, 75, 3)))
 gmodel.add(BatchNormalization())
 gmodel.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2)))
 gmodel.add(Dropout(drop))
gmodel.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
gmodel.add(BatchNormalization())
 gmodel.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
 gmodel.add(Dropout(drop))
gmodel.add(Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same'))
gmodel.add(BatchNormalization())
 gmodel.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'))
gmodel.add(Dropout(drop))
gmodel.add(Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same'))
gmodel.add(BatchNormalization())
 gmodel.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'))
gmodel.add(Dropout(drop))
gmodel.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
gmodel.add(BatchNormalization())
 gmodel.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
 gmodel.add(Dropout(drop))
gmodel.add(Flatten())
 gmodel.add(Dense(512))
gmodel.add(BatchNormalization())
 gmodel.add(Activation('relu'))
gmodel.add(Dropout(drop))
gmodel.add(Dense(256))
gmodel.add(BatchNormalization())
 gmodel.add(Activation('relu'))
gmodel.add(Dropout(drop))
gmodel.add(Dense(1))
gmodel.add(Activation('sigmoid'))
mypotim=Adam(lr=0.0008, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0, amsgrad=True)
 gmodel.compile(loss='binary_crossentropy',
optimizer=mypotim,
metrics=['accuracy'])
 gmodel.summary()
return gmodel
def get_callbacks(filepath, patience=2):
es = EarlyStopping('val_loss', patience=patience, mode="min")
 msave = ModelCheckpoint(filepath, save_best_only=True)
 return [es, msave]
file_path = "model_weights3.hdf5"
callbacks = get_callbacks(filepath=file_path, patience=5)
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 30,
"learning_rate" : 0.01,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.7,
"bagging_frequency" : 5,
"bagging_seed" : 2018,
"verbosity" : -1
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=200, evals_result=evals_result)
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 return pred_test_y, model, evals_result<split>
X_train_cv, X_valid, y_train_cv, y_valid = train_test_split(X_train, y_train, random_state=1, test_size=0.25 )
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 pred_test_full = 0
for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_X.loc[dev_index,:], train_X.loc[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, test_X)
 pred_test_full += pred_test
pred_test_full /= 5.
 pred_test_full = np.expm1(pred_test_full )<save_to_csv>
gen = ImageDataGenerator(rotation_range=10, width_shift_range=0.3, height_shift_range=0.3,
zoom_range=0.2, horizontal_flip=True, vertical_flip=True)
 gen.fit(X_train_cv )
sub_df = pd.DataFrame({"ID":test_df["ID"].values})
 sub_df["target"] = pred_test_full
sub_df.to_csv("baseline_lgb.csv", index=False )<train_model>
gmodel=getModel()
history = gmodel.fit_generator(gen.flow(X_train_cv, y_train_cv, batch_size=32),steps_per_epoch=256,
epochs=50, verbose=1, validation_data=(X_valid, y_valid), callbacks=callbacks )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 col = [c for c in train.columns if c not in ['ID', 'target']]
print(train.shape, test.shape)
 scl = preprocessing.StandardScaler()
def rmsle(y, pred):
return np.sqrt(np.mean(np.power(np.log1p(y)-np.log1p(pred), 2)))
 x1, x2, y1, y2 = model_selection.train_test_split(train[col], train.target.values, test_size=0.20, random_state=5)
 model = ensemble.RandomForestRegressor(n_jobs = -1, random_state = 7)
 model.fit(scl.fit_transform(x1), y1)
 print(rmsle(y2, model.predict(scl.transform(x2))))
col = pd.DataFrame({'importance': model.feature_importances_, 'feature': col} ).sort_values(by=['importance'], ascending=[False])[:480]['feature'].values
test['target_lgb'] = 0.0
test['target_xgb'] = 0.0
folds = 5
for fold in range(folds):
x1, x2, y1, y2 = model_selection.train_test_split(train[col], np.log1p(train.target.values), test_size=0.20, random_state=fold)
 params = {'learning_rate': 0.02,'max_depth': 13,'reg_alpha':0.04,'reg_lambda':0.073,'boosting': 'gbdt','objective': 'regression','metric': 'rmse','is_training_metric': True, 'num_leaves': 12**2,'feature_fraction': 0.9,'bagging_fraction': 0.8, 'bagging_freq': 5,'min_split_gain':0.0222415,'min_child_weight':40,'subsample':0.8715623,'seed':fold,'silent':-1,'verbose':-1}
model = lgb.train(params, lgb.Dataset(x1, label=y1), 3000, lgb.Dataset(x2, label=y2), verbose_eval=200, early_stopping_rounds=100)
 test['target_lgb'] += np.expm1(model.predict(test[col], num_iteration=model.best_iteration))
watchlist = [(xgb.DMatrix(x1, y1), 'train'),(xgb.DMatrix(x2, y2), 'valid')]
params = {'objective': 'reg:linear', 'eval_metric': 'rmse', 'eta': 0.005, 'max_depth': 10, 'subsample': 0.7, 'colsample_bytree': 0.5, 'alpha':0, 'silent': True, 'random_state':fold}
model = xgb.train(params, xgb.DMatrix(x1, y1), 5000, watchlist, maximize=False, verbose_eval=200, early_stopping_rounds=100)
 test['target_xgb'] += np.expm1(model.predict(xgb.DMatrix(test[col]), ntree_limit=model.best_ntree_limit))
test['target_lgb'] /= folds
test['target_xgb'] /= folds
test['target'] =(test['target_lgb'] + test['target_xgb'])/2
test[['ID', 'target']].to_csv('submission1.csv', index=False )<load_from_csv>
gmodel.load_weights(filepath=file_path)
 score = gmodel.evaluate(X_valid, y_valid, verbose=1)
 print('Test loss:', score[0])
 print('Test accuracy:', score[1] )
print("Load data...")
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 subm = pd.read_csv('.. /input/sample_submission.csv')
 print("Train shape: {}
Test shape: {}".format(train.shape, test.shape))
train = train[['ID', 'target']+list(col)]
test = test[['ID']+list(col)]
print("Train shape: {}
Test shape: {}".format(train.shape, test.shape))
PERC_TRESHOLD = 0.98
N_COMP = 20
target = np.log1p(train['target'] ).values
cols_to_drop = [col for col in train.columns[2:]
if [i[1] for i in list(train[col].value_counts().items())
 if i[0] == 0][0] >= train.shape[0] * PERC_TRESHOLD]
print("Define training features...")
 exclude_other = ['ID', 'target']
train_features = []
for c in train.columns:
if c not in cols_to_drop \
and c not in exclude_other:
train_features.append(c)
 print("Number of featuress for training: %s" % len(train_features))
train, test = train[train_features], test[train_features]
print("
Train shape: {}
Test shape: {}".format(train.shape, test.shape))
print("
Start decomposition process...")
 print("PCA")
 pca = PCA(n_components=N_COMP, random_state=17)
 pca_results_train = pca.fit_transform(train)
 pca_results_test = pca.transform(test)
 print("tSVD")
 tsvd = TruncatedSVD(n_components=N_COMP, random_state=17)
 tsvd_results_train = tsvd.fit_transform(train)
 tsvd_results_test = tsvd.transform(test)
 print("ICA")
 ica = FastICA(n_components=N_COMP, random_state=17)
 ica_results_train = ica.fit_transform(train)
 ica_results_test = ica.transform(test)
 print("GRP")
 grp = GaussianRandomProjection(n_components=N_COMP, eps=0.1, random_state=17)
 grp_results_train = grp.fit_transform(train)
 grp_results_test = grp.transform(test)
 print("SRP")
 srp = SparseRandomProjection(n_components=N_COMP, dense_output=True, random_state=17)
 srp_results_train = srp.fit_transform(train)
 srp_results_test = srp.transform(test)
 print("Append decomposition components to datasets...")
 for i in range(1, N_COMP + 1):
train['pca_' + str(i)] = pca_results_train[:, i - 1]
test['pca_' + str(i)] = pca_results_test[:, i - 1]
train['ica_' + str(i)] = ica_results_train[:, i - 1]
test['ica_' + str(i)] = ica_results_test[:, i - 1]
train['tsvd_' + str(i)] = tsvd_results_train[:, i - 1]
test['tsvd_' + str(i)] = tsvd_results_test[:, i - 1]
train['grp_' + str(i)] = grp_results_train[:, i - 1]
test['grp_' + str(i)] = grp_results_test[:, i - 1]
train['srp_' + str(i)] = srp_results_train[:, i - 1]
test['srp_' + str(i)] = srp_results_test[:, i - 1]
print('
Train shape: {}
Test shape: {}'.format(train.shape, test.shape))
print('
Modelling...')
 def rmsle(y_true, y_pred):
assert len(y_true)== len(y_pred)
 return np.sqrt(np.mean(np.power(np.log(y_true + 1)- np.log(y_pred + 1), 2)))
 folds = KFold(n_splits=5, shuffle=True, random_state=546789)
 oof_preds = np.zeros(train.shape[0])
 sub_preds = np.zeros(test.shape[0])
 for n_fold,(trn_idx, val_idx)in enumerate(folds.split(train)) :
trn_x, trn_y = train.ix[trn_idx], target[trn_idx]
val_x, val_y = train.ix[val_idx], target[val_idx]
cb_model = CatBoostRegressor(iterations=1000, learning_rate=0.1, depth=8, l2_leaf_reg=20, bootstrap_type='Bernoulli', eval_metric='RMSE', metric_period=50, od_type='Iter', od_wait=45, random_seed=17, allow_writing_files=False)
 cb_model.fit(trn_x, trn_y, eval_set=(val_x, val_y), cat_features=[], use_best_model=True, verbose=True)
 oof_preds[val_idx] = cb_model.predict(val_x)
 sub_preds += cb_model.predict(test)/ folds.n_splits
print("Fold %2d RMSLE : %.6f" %(n_fold+1, rmsle(np.exp(val_y)-1, np.exp(oof_preds[val_idx])-1)))
 print("Full RMSLE score %.6f" % rmsle(np.exp(target)-1, np.exp(oof_preds)-1))
subm['target'] = np.exp(sub_preds)-1
subm.to_csv('submission2.csv', index=False )<save_to_csv>
pred = np.round(gmodel.predict_proba(X_valid[0:3,:,:,:]))
for i in range(3):
print('real:', y_valid[i], 'predicted:', pred[i])
 plt.imshow(X_valid[i,:,:,0])
 plt.show()
<set_options><EOS> 
gmodel=getModel()
gmodel.load_weights(filepath=file_path)
 test = pd.read_json(".. /input/test.json")
 X_test = process_data(test, predict=True)
 predicted_test=gmodel.predict_proba(X_test)
 submission = pd.DataFrame()
submission['id'] = test['id']
submission['is_iceberg'] = predicted_test.reshape(( predicted_test.shape[0]))
submission.to_csv('sub3.csv', index=False)
 del test
del X_test