Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<define_variables>
warnings.simplefilter(action='ignore', category=FutureWarning )
class LungSliceModelGenerator(kutils.Sequence):
'Generates data for Keras'
def __init__(self, mapping_df, batch_size, shuffle=True):
'Initialization'
self.mapping_df = mapping_df
self.data_num = mapping_df.shape[0]
self.batch_size = batch_size
self.shuffle = shuffle
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(self.data_num / self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
batch_mapping_df = \
self.mapping_df.iloc[index*self.batch_size:(index+1)*self.batch_size]
X, y = self.__data_generation(batch_mapping_df)
 return X, y
def on_epoch_end(self):
'Updates indexes after each epoch'
if self.shuffle:
self.mapping_df = self.mapping_df.sample(frac=1 ).reset_index(drop=True)
 def __data_generation(self, batch_mapping_df):
'Generates data containing batch_size samples'
X = np.zeros(( self.batch_size, 512, 512, 1))
y = np.zeros(( self.batch_size, 512, 512, 1))
cnt = 0
for i, row in batch_mapping_df.iterrows() :
X[cnt, :, :, 0] = np.load(row['image'])['image']
y[cnt, :, :, 0] = np.load(row['label'])['label']
cnt += 1
return X, y<choose_model_class>
df = pd.read_csv(".. /input/dfcsv/df.csv" )
batch_size = 16
slice_generator = LungSliceModelGenerator(map_df, batch_size=batch_size )<compute_test_metric>
df_model = df[df['TARGET'].notnull() ]
feats = [f for f in df_model.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]
train_x, test_x, train_y, test_y = train_test_split(df_model[feats], df_model['TARGET'], random_state=42)
 df_submission = df.loc[df['TARGET'].isnull() , feats]
main_id_submission =df.loc[df['TARGET'].isnull() , 'SK_ID_CURR']
del df
def _dice_coefficient(threshold = 0.3):
def hard_dice_coefficient(y_true, y_pred, smooth=1.0):
y_true_f = K.flatten(K.cast(y_true > threshold, dtype=float))
y_pred_f = K.flatten(K.cast(y_pred > threshold, dtype=float))
intersection = K.sum(y_true_f * y_pred_f)
 return(2.* intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return hard_dice_coefficient
def dice_coefficient_loss(y_true, y_pred):
return 1 - _dice_coefficient()(y_true, y_pred )<choose_model_class>
DEBUG = False
def unet(pretrained_weights=None, input_size=[512, 512, 1], depth=3, init_filter=8,
filter_size=3, padding='same', pool_size=[2, 2], strides=[2, 2]):
inputs = klayers.Input(input_size)
 current_layer = inputs
encoding_layers = []
for d in range(depth + 1):
num_filters = init_filter * 2 ** d
conv = klayers.Conv2D(num_filters, filter_size, padding=padding, kernel_initializer='he_normal' )(current_layer)
 conv = klayers.BatchNormalization()(conv)
 conv = klayers.Activation('relu' )(conv)
 conv = klayers.Conv2D(num_filters * 2, filter_size, padding=padding, kernel_initializer='he_normal' )(conv)
 conv = klayers.BatchNormalization()(conv)
 conv = klayers.Activation('relu' )(conv)
 encoding_layers.append(conv)
 pool = klayers.MaxPooling2D(pool_size=pool_size )(conv)
 if d == depth:
current_layer = conv
else:
current_layer = pool
for d in range(depth, 0, -1):
num_filters = init_filter * 2 ** d
up = klayers.Deconvolution2D(num_filters * 2, pool_size, strides=strides )(current_layer)
 crop_layer = encoding_layers[d - 1]
up_shape = np.array(up._keras_shape[1:-1])
 conv_shape = np.array(crop_layer._keras_shape[1:-1])
 crop_left =(conv_shape - up_shape)// 2
crop_right =(conv_shape - up_shape)// 2 +(conv_shape - up_shape)% 2
crop_sizes = tuple(zip(crop_left, crop_right))
crop = klayers.Cropping2D(cropping=crop_sizes )(crop_layer)
 up = klayers.Concatenate(axis=-1 )([crop, up])
 conv = klayers.Conv2D(num_filters, filter_size, padding=padding, kernel_initializer='he_normal' )(up)
 conv = klayers.BatchNormalization()(conv)
 conv = klayers.Activation('relu' )(conv)
 conv = klayers.Conv2D(num_filters, filter_size, padding=padding, kernel_initializer='he_normal' )(conv)
 conv = klayers.BatchNormalization()(conv)
 conv = klayers.Activation('relu' )(conv)
 current_layer = conv
outputs = klayers.Conv2D(1, 1, padding=padding, kernel_initializer='he_normal' )(current_layer)
 outputs = klayers.Activation('sigmoid' )(outputs)
 model = Model(inputs=inputs, outputs=outputs)
 if(pretrained_weights):
model.load_weights(pretrained_weights)
 return model<choose_model_class>
ITER = 1
SCORES = []
MINUTES = time.time()
if DEBUG == True:
init_pt = 1
n_iter_pt = 2
PT_GRAPH = 3
else:
init_pt = 10
n_iter_pt = 100
PT_GRAPH = 10
def lgb_evaluate(
 numLeaves,
maxDepth,
minChildWeight,
subsample,
colsample_bytree,
learn_rate,
reg_alpha,
reg_lambda,
min_split_gain):
global ITER, SCORES, MINUTES
clf = LGBMClassifier(
 nthread=4,
n_estimators=100,
verbose =-1,
silent=-1,
num_leaves= int(numLeaves),
max_depth= int(maxDepth),
min_child_weight= minChildWeight,
colsample_bytree= colsample_bytree,
subsample= subsample,
learning_rate= learn_rate,
reg_alpha = reg_alpha,
reg_lambda= reg_lambda,
min_split_gain= min_split_gain 
)
 scores = cross_val_score(clf, train_x, train_y, cv=5, scoring='roc_auc')
 print("Mean cross validation score: {}".format(np.mean(scores)))
 SCORES.append(np.mean(scores))
if ITER % PT_GRAPH == 0:
plt.figure(figsize=(11,4))
plt.plot(range(len(SCORES)) , SCORES)
 plt.scatter(SCORES.index(max(SCORES)) , max(SCORES), color='red')
 plt.ylabel("Score")
 plt.xlabel("Attempt")
 plt.title("Real time evolution of the mean score")
 plt.show()
print("Minutes since beginning: {}".format(float(time.time() - MINUTES)/ 60))
ITER = ITER + 1
return np.mean(scores)
 lgbBO = BayesianOptimization(lgb_evaluate, {
'numLeaves':(5, 50),
'maxDepth':(2, 63),
'minChildWeight':(0.01, 70),
'subsample':(0.4, 1),
'colsample_bytree':(0.4, 1),
'learn_rate':(0.1, 1),
'reg_alpha':(0, 1),
'reg_lambda':(0, 1),
'min_split_gain':(0, 1)
 })
 lgbBO.maximize(init_points=init_pt, n_iter=n_iter_pt )
model = unet(depth=3)
 model.compile(optimizer=Adam(lr=1e-3), loss='binary_crossentropy', metrics=[_dice_coefficient(0.5)])
 model.summary()<train_model>
best = max([lgbBO.res[i]['target'] for i in range(len(lgbBO.res)) ])
 best
model_folder = os.path.join('./model', 'sample-code')
 if not os.path.exists(model_folder):
os.makedirs(model_folder)
 callbacks = []
callbacks.append(ModelCheckpoint(os.path.join(model_folder, 'model-{epoch:03d}.h5'),
save_best_only=False,
period=5))<train_model>
best_index = [lgbBO.res[i]['target'] for i in range(len(lgbBO.res)) ].index(best)
 best_index
history = model.fit_generator(slice_generator,
epochs=15,
verbose=1,
callbacks=callbacks )<predict_on_test>
param_dict = lgbBO.res[best_index]["params"]
clf = LGBMClassifier(
 nthread=4,
n_estimators=100,
silent=-1,
verbose=-1,
num_leaves=34,
colsample_bytree=param_dict["colsample_bytree"],
subsample=param_dict["subsample"],
max_depth=int(param_dict["maxDepth"]),
min_child_weight=param_dict["minChildWeight"],
learning_rate=param_dict["learn_rate"],
reg_alpha=param_dict["reg_alpha"],
reg_lambda=param_dict["reg_lambda"],
min_split_gain=param_dict["min_split_gain"])
 clf.fit(train_x, train_y )
def retrieve_pred_str(src_dir, model, threshold=0.4):
encode_name = src_dir.split('/')[-1]
_, test_volume = load_dicom_volume(src_dir, suffix='*.dcm')
 pred_label = model.predict(np.expand_dims(test_volume, axis=-1))
pred_label = np.transpose(pred_label[:, :, :, 0], axes=(2, 1, 0))
pred_label =(pred_label > threshold ).astype(np.int)
 label_flatten = pred_label.flatten()
label_flatten_idx = np.where(label_flatten == 1)[0]
label_str = ''
if label_flatten_idx.size > 0:
prev_idx = label_flatten_idx[0]
idx_start = label_flatten_idx[0]
cnt = 1
for _idx in label_flatten_idx[1:]:
if _idx == prev_idx+1:
cnt += 1
else:
label_str += str(idx_start)+ ' ' + str(cnt)+ ' '
cnt = 1
idx_start = _idx
prev_idx = _idx
label_str = label_str.rstrip(' ')
 return(encode_name, label_str )<load_from_csv>
print(metrics.auc(fpr, tpr))
sample_submission = np.genfromtxt('.. /input/sample_submission.csv',
delimiter=',',
dtype='str',
skip_header = 1 )<define_variables>
importance_df = pd.DataFrame()
importance_df["feature"] = feats
importance_df["importance"] = clf.feature_importances_
importance_df = importance_df.sort_values(by='importance', ascending=False)
 importance_df = importance_df.reset_index(drop=True )
test_encode_list = sample_submission[:, 0]<categorify>
display_importances(importance_df )
pred_pair_list = []
for encode_name in tqdm.tqdm(test_encode_list, total=len(test_encode_list)) : 
(encode, label_str)= retrieve_pred_str(os.path.join(test_image_folder, encode_name), model, threshold=0.4)
 pred_pair_list.append(( encode, label_str))<save_to_csv>
best_feature = importance_df.loc[0:30, "feature"].values
solution_path = './sample-code_pred.csv'
with open(solution_path, 'w')as f:
f.write('encode,pixel_value
')
 for _pair in pred_pair_list:
encode = _pair[0]
label_str = _pair[1]
f.write(encode + ',' + label_str + '
' )<import_modules>
values_x = pd.concat([train_x, test_x])
 values_y = pd.concat([train_y, test_y] )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import keras
from sklearn import metrics
from sklearn.preprocessing import LabelEncoder,OneHotEncoder
from keras.models import Model
from keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding
from keras.optimizers import RMSprop
from keras.preprocessing.text import Tokenizer
from keras.preprocessing import sequence
from keras.callbacks import EarlyStopping<import_modules>
param_dict = lgbBO.res[best_index]["params"]
clf = LGBMClassifier(
 nthread=4,
n_estimators=100,
silent=-1,
verbose=-1,
num_leaves=34,
colsample_bytree=param_dict["colsample_bytree"],
subsample=param_dict["subsample"],
max_depth=int(param_dict["maxDepth"]),
min_child_weight=param_dict["minChildWeight"],
learning_rate=param_dict["learn_rate"],
reg_alpha=param_dict["reg_alpha"],
reg_lambda=param_dict["reg_lambda"],
min_split_gain=param_dict["min_split_gain"])
 clf.fit(values_x, values_y )
import json
from pandas.io.json import json_normalize<load_from_csv>
filename = 'clf.sav'
pickle.dump(clf, open(filename, 'wb'))
raw_data=pd.read_json(".. /input/datamininglab2/tweets_DM.json",lines=True)
 tweets=json_normalize(data=raw_data['_source'])
 identify=pd.read_csv(".. /input/datamininglab2/data_identification.csv")
 emotion=pd.read_csv(".. /input/datamininglab2/emotion.csv" )<merge>
y_pred_proba = clf.predict_proba(df_submission)[:, 1]
df_results = pd.DataFrame(columns =['SK_ID_CURR', 'TARGET'])
 df_results['SK_ID_CURR'] = main_id_submission
df_results['TARGET'] = y_pred_proba
<load_pretrained><EOS> 
df_results.to_csv("submission.csv", index=False )