Kaggle competition: Statoil/C-CORE Iceberg Classifier Challenge
<SOS> metric: LogLoss Kaggle data source: statoil-iceberg-classifier-challenge<load_from_csv>
ANGLE = "inc_angle"
ICEBERG = "is_iceberg"
BAND1 = "band_1"
BAND2 = "band_2"
ID = "id"
initial_model_path = "lgbm"
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
train = pd.read_json('.. /input/train.json')
 print("rows, cols = " + str(train.shape))
train.head()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<define_search_space>
train[ANGLE] = pd.to_numeric(train[ANGLE],errors='coerce')
 nullPD = pd.DataFrame(train.isnull().sum() , columns=['TotalNull'])
 nullPD
pd.Series(unique_values ).quantile([0.25, 0.50, 0.75] )<count_values>
P = train.groupby(ICEBERG)[ID].count().reset_index()
P['Percentage'] = 100*P[ID]/P[ID].sum()
P
pd.Series(unique_values ).value_counts()<count_values>
MIN="min"
MAX="max"
MEAN="mean"
STD="std"
LAPLACE="laplacestd"
SOBEL0 = "sobelstd_x"
SOBEL1 = "sobelstd_y"
KURTOSIS = "kurtosis"
SKEW = "skew"
CORR = "pearson"
HYP = "hypstd"
AGG_COLS = [
"%s_%s"%(BAND1,MAX), "%s_%s"%(BAND2,MAX),
"%s_%s"%(BAND1,MIN), "%s_%s"%(BAND2,MIN),
"%s_%s"%(BAND1,MEAN), "%s_%s"%(BAND2,MEAN),
"%s_%s"%(BAND1,STD), "%s_%s"%(BAND2,STD),
"%s_%s"%(BAND1,KURTOSIS), "%s_%s"%(BAND2,KURTOSIS),
"%s_%s"%(BAND1,SKEW), "%s_%s"%(BAND2,SKEW),
"%s_%s"%(BAND1,SOBEL0), "%s_%s"%(BAND2,SOBEL0),
"%s_%s"%(BAND1,SOBEL1), "%s_%s"%(BAND2,SOBEL1),
"%s_%s"%(BAND1,LAPLACE), "%s_%s"%(BAND2,LAPLACE),
CORR,
HYP,
]
ISO = "iso"
ISO_COLS = [
"%s_%s"%(BAND1,ISO), "%s_%s"%(BAND2,ISO)
 ]
VOLUME = "vol"
FEATURES = [ANGLE, VOLUME] + AGG_COLS
def iso(arr):
image = img_as_float(np.reshape(np.array(arr), [75,75]))
image = gaussian_filter(image,2.5)
 seed = np.copy(image)
 seed[1:-1, 1:-1] = image.min()
mask = image
dilated = reconstruction(seed, mask, method='dilation')
 return image-dilated
def sobelstd(arr, axis=0):
image = img_as_float(np.reshape(np.array(arr), [75,75]))
sobelstd = sobel(image, axis=axis, mode='reflect', cval=0.0 ).ravel()
return [sobelstd.std() , sobelstd.max() , sobelstd.mean() ]
def lapacestd(arr):
image = img_as_float(np.reshape(np.array(arr), [75,75]))
lapacestd = laplace(image, mode='reflect', cval=0.0 ).ravel()
return [lapacestd.std() , lapacestd.max() , lapacestd.mean() ]
def volume(arr):
return np.sum(arr)
 def hypot(arr1, arr2):
hyp = np.hypot(arr1, arr2)
 return [np.std(hyp), np.max(hyp), np.median(hyp)]
def computeAdditionalFeatures(df):
df["%s_%s"%(BAND1,MAX)] = df[BAND1].apply(lambda x: np.max(x))
df["%s_%s"%(BAND2,MAX)] = df[BAND2].apply(lambda x: np.max(x))
df["%s_%s"%(BAND1,MIN)] = df[BAND1].apply(lambda x: np.min(x))
df["%s_%s"%(BAND2,MIN)] = df[BAND2].apply(lambda x: np.min(x))
df["%s_%s"%(BAND1,MEAN)] = df[BAND1].apply(lambda x: np.mean(x))
df["%s_%s"%(BAND2,MEAN)] = df[BAND2].apply(lambda x: np.mean(x))
df["%s_%s"%(BAND1,STD)] = df[BAND1].apply(lambda x: np.std(x))
df["%s_%s"%(BAND2,STD)] = df[BAND2].apply(lambda x: np.std(x))
df["%s_%s"%(BAND1,KURTOSIS)] = df[BAND1].apply(lambda x: kurtosis(x))
df["%s_%s"%(BAND2,KURTOSIS)] = df[BAND2].apply(lambda x: kurtosis(x))
df["%s_%s"%(BAND1,SKEW)] = df[BAND1].apply(lambda x: skew(x))
df["%s_%s"%(BAND2,SKEW)] = df[BAND2].apply(lambda x: skew(x))
df["%s_%s"%(BAND1,SOBEL0)] = df[BAND1].apply(lambda x: sobelstd(x, axis=0)[0])
 df["%s_%s"%(BAND1,SOBEL1)] = df[BAND1].apply(lambda x: sobelstd(x, axis=1)[0])
 df["%s_%s"%(BAND2,SOBEL0)] = df[BAND2].apply(lambda x: sobelstd(x, axis=0)[0])
 df["%s_%s"%(BAND2,SOBEL1)] = df[BAND2].apply(lambda x: sobelstd(x, axis=1)[0])
 df["%s_%s"%(BAND1,LAPLACE)] = df[BAND1].apply(lambda x: lapacestd(x)[0])
 df["%s_%s"%(BAND2,LAPLACE)] = df[BAND2].apply(lambda x: lapacestd(x)[0])
 df[CORR] = df.apply(lambda row: np.corrcoef(x=row[BAND1], y=row[BAND2])[1,0], axis=1)
 df[HYP] = df.apply(lambda row: hypot(row[BAND1], row[BAND2])[0], axis=1)
 df["%s_%s"%(BAND1,ISO)] = df[BAND1].apply(lambda x: iso(x))
df["%s_%s"%(BAND2,ISO)] = df[BAND2].apply(lambda x: iso(x))
df[VOLUME] =(df["%s_%s"%(BAND1,ISO)] + df["%s_%s"%(BAND2,ISO)] ).apply(volume)
 cleanDF = df.dropna()
ret = cleanDF[FEATURES].copy(deep=True)
 ret_labels = None
if ICEBERG in cleanDF.columns:
ret_labels = cleanDF[[ICEBERG]]
ret_ids = cleanDF[[ID]]
ret_cols = ret.columns
return ret, ret_labels, ret_ids
zero_count = []
for col in [col for col in train.columns if len(train[col].unique())== 2]:
zero_count.append([i[1] for i in list(train[col].value_counts().items())if i[0] == 0][0])
 print('{0} features of 245 having 2 unique values have zeroes in 99% or more samples.'.format(len([i for i in zero_count if i >= 4459 * 0.99])) )<count_values>
def read_and_normalize_train_data(train_df):
featuresDF, labelsDF, idsDF = computeAdditionalFeatures(train_df.copy(deep=True))
train_features = featuresDF.as_matrix()
train_target = labelsDF.as_matrix()
train_id = idsDF.as_matrix()
print("Features size: %s/%s"%(str(train_features.shape), str(train_target.shape)))
 return train_features, train_target, train_id, featuresDF[FEATURES].columns
zero_count = []
for col in train.columns[2:]:
zero_count.append([i[1] for i in list(train[col].value_counts().items())if i[0] == 0][0])
 print('{0} features of 4491 have zeroes in 99% or more samples.'.format(len([i for i in zero_count if i >= 4459 * 0.99])))
 print('{0} features of 4491 have zeroes in 98% or more samples.'.format(len([i for i in zero_count if i >= 4459 * 0.98])))
 print('{0} features of 4491 have zeroes in 97% or more samples.'.format(len([i for i in zero_count if i >= 4459 * 0.97])))
 print('{0} features of 4491 have zeroes in 96% or more samples.'.format(len([i for i in zero_count if i >= 4459 * 0.96])))
 print('{0} features of 4491 have zeroes in 95% or more samples.'.format(len([i for i in zero_count if i >= 4459 * 0.95])) )<count_values>
def fit_evaluate_model_lgbm(X_train, Y_train, X_valid, Y_valid, train_data_columns, model_path, num_fold, importance=False):
X_trainDF = pd.DataFrame(X_train, columns=train_data_columns)
 X_validDF = pd.DataFrame(X_valid, columns=train_data_columns)
 train_dataset = lgb.Dataset(X_trainDF, Y_train.reshape(Y_train.shape[0]))
test_dataset = lgb.Dataset(X_validDF, Y_valid.reshape(Y_valid.shape[0]))
evals_result = {}
params = {
'objective': 'binary',
'metric': 'binary_logloss',
'boosting': 'gbdt',
'learning_rate': 0.1,
'num_rounds': 150,
'early_stopping_rounds': 100
}
gbm = lgb.train(params, train_dataset,
valid_sets=test_dataset,
evals_result=evals_result,
verbose_eval=50)
 predict_y_proba_gbm = gbm.predict(X_valid, num_iteration=gbm.best_iteration)
 predict_y_gbm = np.where(predict_y_proba_gbm.reshape(( predict_y_proba_gbm.shape[0])) > 0.5, 1, 0)
 score_ll = metrics.log_loss(Y_valid, predict_y_proba_gbm)
 score_ac = metrics.accuracy_score(Y_valid, predict_y_gbm)
 score_pr = metrics.precision_score(Y_valid, predict_y_gbm)
 score_re = metrics.recall_score(Y_valid, predict_y_gbm)
 score = [score_ll, score_ac, score_pr, score_re]
if(importance == True):
ax = lgb.plot_importance(gbm, max_num_features=20, figsize=(16, 5))
plt.show()
gbmDF = pd.DataFrame([tuple(gbm.feature_importance())], columns= gbm.feature_name())
 gbmDF.sort_index(axis=1, inplace=True)
 return score, gbmDF, gbm
cols_to_drop = [col for col in train.columns[2:] if [i[1] for i in list(train[col].value_counts().items())if i[0] == 0][0] >= 4459 * 0.98]<drop_column>
def run_cross_validation_create_models(train_df, model_path, nfolds=4, break_fold=-1, importance=False):
train_data, train_target, train_id, train_data_columns = read_and_normalize_train_data(train_df)
 train_target = train_target.ravel()
kf = StratifiedKFold(n_splits=nfolds, random_state=None, shuffle=True)
 num_fold = 0
sum_score_ll = 0
sum_score_ac = 0
sum_score_pr = 0
sum_score_re = 0
scores = []
models = []
importanceDF = pd.DataFrame()
for train_index, test_index in kf.split(train_data, train_target):
num_fold += 1
print('
==> Start KFold number {} from {}'.format(num_fold, nfolds))
X_train = train_data[train_index]
Y_train = train_target[train_index]
X_valid = train_data[test_index]
Y_valid = train_target[test_index]
print("Train size: %s/%s"%(str(X_train.shape), str(Y_train.shape)))
 print("Valid size: %s/%s"%(str(X_valid.shape), str(Y_valid.shape)))
 score, impDF, m = fit_evaluate_model_lgbm(X_train,Y_train, X_valid, Y_valid, train_data_columns,
model_path, num_fold, importance=False)
 models.append(m)
 if len(importanceDF)== 0:
importanceDF = impDF
else:
importanceDF = pd.concat([importanceDF, impDF])
 print('Test loss:', score[0])
 print('Test accuracy:', score[1])
 sum_score_ll += score[0]*len(test_index)
 sum_score_ac += score[1]*len(test_index)
 sum_score_pr += score[2]*len(test_index)
 sum_score_re += score[3]*len(test_index)
 scores.append(score)
 if(break_fold > 0)&(break_fold == num_fold):
break
score_ll = sum_score_ll/len(train_data)
 score_ac = sum_score_ac/len(train_data)
 score_pr = sum_score_pr/len(train_data)
 score_re = sum_score_re/len(train_data)
 print('
CV average scores:')
 print('log_loss: %s
accuracy: %s
precision: %s
recall: %s
'%(score_ll, score_ac, score_pr, score_re))
return scores, importanceDF, models
train.drop(cols_to_drop, axis=1, inplace=True)
 test.drop(cols_to_drop, axis=1, inplace=True )<prepare_x_and_y>
test = pd.read_json('.. /input/test.json')
 print("rows, cols = " + str(test.shape))
X = train.drop(['ID', 'target'], axis=1)
 y = train['target']
X_test = test.drop('ID', axis=1 )<choose_model_class>
def read_and_normalize_test_data(test_df):
featuresDF, labelsDF, idsDF = computeAdditionalFeatures(test_df.copy(deep=True))
test_features = featuresDF.as_matrix()
test_id = idsDF.as_matrix()
print("Features size: %s/%s"%(str(test_features.shape), str(test_id.shape)))
 return test_features, test_id, featuresDF[FEATURES].columns
ridge = Ridge()<compute_test_metric>
X_test, X_test_id, X_test_columns = read_and_normalize_test_data(test )
def rmsle(y_true, y_pred):
return np.sqrt(np.mean(np.power(np.log1p(y_true)- np.log1p(y_pred), 2)))
 rmsle_scorer = make_scorer(rmsle, greater_is_better=False )<compute_test_metric>
yfull_proba_train = []
yfull_proba_test = []
yfull_label_test = []
num_fold = 0
for model in models:
num_fold += 1
print('==> Start KFold number {} from {}'.format(num_fold, NFOLDS))
predicted_test = model.predict(X_test, num_iteration=model.best_iteration)
 predicted_test_label = np.where(predicted_test.reshape(( predicted_test.shape[0])) > 0.5, 1, 0)
 yfull_proba_test.append(predicted_test)
 yfull_label_test.append(predicted_test_label )
-cross_val_score(ridge, X, y, scoring=rmsle_scorer )<train_model>
def merge_several_folds_mean(data, nfolds):
a = np.array(data[0])
 aPD = pd.DataFrame(a, columns = ["%s_%d"%(ICEBERG, 1)])
 for i in range(1, nfolds):
a = a + np.array(data[i])
 aPD["%s_%d"%(ICEBERG, i+1)] = np.array(data[i])
 a = a / nfolds*1.0
return a, aPD
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.20, random_state=42)
 params = {'learning_rate': 0.01, 'max_depth': 6, 'boosting': 'gbdt', 'objective': 'regression', 'metric': ['rmse'], 'is_training_metric': True, 'seed': 19, 'num_leaves': 63, 'feature_fraction': 0.9, 'bagging_fraction': 0.8, 'bagging_freq': 5}
model = lgb.train(params, lgb.Dataset(X_train, label=y_train), 2000, lgb.Dataset(X_valid, label=y_valid), verbose_eval=50, early_stopping_rounds=20)
 print('RMSLE on valid data: {0:.4}.'.format(rmsle(y_valid, model.predict(X_valid))))<train_model>
kfold_cols = ["%s_%d"%(ICEBERG, i)for i in range(1, NFOLDS + 1)]
predicted_test_mean, predicted_test_pd = merge_several_folds_mean(yfull_proba_test, NFOLDS)
 predicted_test_pd.head()
X_train, X_valid, y_train, y_valid = train_test_split(X, np.log1p(y), test_size=0.20, random_state=42)
 params = {'learning_rate': 0.01, 'max_depth': 6, 'boosting': 'gbdt', 'objective': 'regression', 'metric': ['rmse'], 'is_training_metric': True, 'seed': 19, 'num_leaves': 63, 'feature_fraction': 0.9, 'bagging_fraction': 0.8, 'bagging_freq': 5}
model = lgb.train(params, lgb.Dataset(X_train, label=y_train), 2000, lgb.Dataset(X_valid, label=y_valid), verbose_eval=50, early_stopping_rounds=20)
 print('RMSLE on valid data: {0:.4}.'.format(rmsle(np.expm1(y_valid), np.expm1(model.predict(X_valid)))) )<train_model>
submission = pd.DataFrame()
submission[ID]=test[ID]
submission[ICEBERG]=predicted_test_mean
submission.to_csv("submissionlgbmv1.csv", index=False, sep=",", decimal="." )