Kaggle competition: Statoil/C-CORE Iceberg Classifier Challenge
<SOS> metric: LogLoss Kaggle data source: statoil-iceberg-classifier-challenge<load_pretrained>
%matplotlib inline
TRAIN_FEATURES = ['id', 'keyword', 'location', 'text', 'target', 'text_cleaned', 'target_relabeled']
TEST_FEATURES = ['id', 'keyword', 'location', 'text', 'target', 'text_cleaned']
df_train = pd.read_pickle('.. /input/test12/train.pkl')
 df_test = pd.read_pickle('.. /input/test12/test.pkl' )<count_unique_values>
train = pd.read_json(".. /input/train.json")
 train['inc_angle'] = pd.to_numeric(train['inc_angle'],errors='coerce')
 test = pd.read_json(".. /input/test.json")
 test['inc_angle'] = pd.to_numeric(test['inc_angle'],errors='coerce')
 train = train[train['inc_angle'].isnull() == False].reset_index(drop = True)
 train['mean'] = np.NaN
train['count'] = np.NaN
test['mean'] = np.NaN
test['count'] = np.NaN
train['neighbor_iceberg'] = np.NaN
test['neighbor_iceberg'] = np.NaN
K = 2
skf = StratifiedKFold(n_splits=K, random_state=SEED, shuffle=True)
 DISASTER = df_train['target'] == 1
print('Whole Training Set Shape = {}'.format(df_train.shape))
print('Whole Training Set Unique keyword Count = {}'.format(df_train['keyword'].nunique()))
print('Whole Training Set Target Rate(Disaster){}/{}(Not Disaster)'.format(df_train[DISASTER]['target_relabeled'].count() , df_train[~DISASTER]['target_relabeled'].count()))
for fold,(trn_idx, val_idx)in enumerate(skf.split(df_train['text_cleaned'], df_train['target']), 1):
print('
Fold {} Training Set Shape = {} - Validation Set Shape = {}'.format(fold, df_train.loc[trn_idx, 'text_cleaned'].shape, df_train.loc[val_idx, 'text_cleaned'].shape))
print('Fold {} Training Set Unique keyword Count = {} - Validation Set Unique keyword Count = {}'.format(fold, df_train.loc[trn_idx, 'keyword'].nunique() , df_train.loc[val_idx, 'keyword'].nunique()))<prepare_x_and_y>
for i in train.index:
mean = train[np.abs(train['inc_angle'] - train.loc[i, 'inc_angle'])< 0.00000001]
count = mean.shape[0] + test[ np.abs(test['inc_angle'] - \
train.loc[i, 'inc_angle'])< 0.00000001 ].shape[0] - 1
train.loc[i, 'count'] = count
try:
mean = mean.drop(i)['is_iceberg'].mean()
except:
mean = np.NaN
if mean == mean:
train.loc[i, 'mean'] = mean
else:
train.loc[i, 'mean'] = 0.5
temp = train[np.abs(train['inc_angle'] - train.loc[i, 'inc_angle'])< 0.025].drop(i)
 train.loc[i,'neighbor_iceberg'] = sum(temp['is_iceberg']==1)*1./(1+temp.shape[0])
 for i in test.index:
mean = train[ np.abs(train['inc_angle'] - test.loc[i, 'inc_angle'])< 0.00000001 ]
count = mean.shape[0] + test[ np.abs(test['inc_angle'] - \
test.loc[i, 'inc_angle'])< 0.00000001 ].shape[0] - 1
test.loc[i,'count'] = count
mean = mean['is_iceberg'].mean()
if mean == mean:
test.loc[i, 'mean'] = mean
else:
test.loc[i, 'mean'] = 0.5
temp = train[np.abs(train['inc_angle'] - test.loc[i, 'inc_angle'])< 0.025]
test.loc[i,'neighbor_iceberg'] = sum(temp['is_iceberg']==1)*1./(1+temp.shape[0] )
class ClassificationReport(Callback):
def __init__(self, train_data=() , validation_data=()):
super(Callback, self ).__init__()
self.X_train, self.y_train = train_data
self.train_precision_scores = []
self.train_recall_scores = []
self.train_f1_scores = []
self.X_val, self.y_val = validation_data
self.val_precision_scores = []
self.val_recall_scores = []
self.val_f1_scores = []
def on_epoch_end(self, epoch, logs={}):
train_predictions = np.round(self.model.predict(self.X_train, verbose=0))
train_precision = precision_score(self.y_train, train_predictions, average='macro')
 train_recall = recall_score(self.y_train, train_predictions, average='macro')
 train_f1 = f1_score(self.y_train, train_predictions, average='macro')
 self.train_precision_scores.append(train_precision)
 self.train_recall_scores.append(train_recall)
 self.train_f1_scores.append(train_f1)
 val_predictions = np.round(self.model.predict(self.X_val, verbose=0))
val_precision = precision_score(self.y_val, val_predictions, average='macro')
 val_recall = recall_score(self.y_val, val_predictions, average='macro')
 val_f1 = f1_score(self.y_val, val_predictions, average='macro')
 self.val_precision_scores.append(val_precision)
 self.val_recall_scores.append(val_recall)
 self.val_f1_scores.append(val_f1)
 print('
Epoch: {} - Training Precision: {:.6} - Training Recall: {:.6} - Training F1: {:.6}'.format(epoch + 1, train_precision, train_recall, train_f1))
print('Epoch: {} - Validation Precision: {:.6} - Validation Recall: {:.6} - Validation F1: {:.6}'.format(epoch + 1, val_precision, val_recall, val_f1))<choose_model_class>
clf = LogisticRegression()
clf.fit(train[['mean', 'count', 'neighbor_iceberg']], train['is_iceberg'] )
<choose_model_class><EOS> 
predictions = pd.DataFrame([])
 predictions['id'] = test['id'].copy()
predictions['is_iceberg'] = clf.predict_proba(test[['mean', 'count', 'neighbor_iceberg']])[:,1]
predictions.to_csv("./submission.csv", index=False )