Kaggle competition: Porto Seguro’s Safe Driver Prediction
<SOS> metric: NormalizedGini Kaggle data source: porto-seguro’s-safe-driver-prediction<load_pretrained>
data_path = '/kaggle/input/porto-seguro-safe-driver-prediction/'
train = pd.read_csv(data_path + 'train.csv', index_col='id')
 test = pd.read_csv(data_path + 'test.csv', index_col='id')
 submission = pd.read_csv(data_path + 'sample_submission.csv', index_col='id' )
test_dcm_fps = list(set(glob.glob(os.path.join(test_dcm_dir, '*.dcm'))))
test_dcm_fps = pd.Series(test_dcm_fps ).apply(lambda dcm_fp: dcm_fp.strip().split("/")[-1].replace(".dcm",""))
save_yolov3_test_data(test_dcm_dir, img_dir, metadata_dir, "te_list.txt", test_dcm_fps )<train_model>
all_data = pd.concat([train, test], ignore_index=True)
 all_data = all_data.drop('target', axis=1)
 all_features = all_data.columns.tolist()
ex_patient_id = test_dcm_fps[0]
ex_img_path = os.path.join(img_dir, "{}.jpg".format(ex_patient_id))
plt.imshow(cv2.imread(ex_img_path))<save_to_csv>
all_data['num_missing'] =(all_data==-1 ).sum(axis=1)
 remaining_features = [col for col in all_features \
if('cat' not in col and 'calc' not in col)]
remaining_features.append('num_missing' )
data_extention_file_path = os.path.join(cfg_dir, 'rsna.data')
 with open(data_extention_file_path, 'w')as f:
contents =.format(os.path.join(metadata_dir, "tr_list.txt"),
os.path.join(metadata_dir, "val_list.txt"),
os.path.join(cfg_dir, 'rsna.names'),
backup_dir)
 f.write(contents )<concatenate>
cat_features = [col for col in all_features if 'cat' in col]
onehot_encoder = OneHotEncoder()
encoded_cat_matrix = onehot_encoder.fit_transform(all_data[cat_features] )
!cat cfg/rsna.data<set_options>
ind_features = [col for col in all_features if 'ind' in col]
first_col=True
for col in ind_features:
if first_col:
all_data['mix_ind'] = all_data[col].astype(str)+'_'
first_col = False
else:
all_data['mix_ind'] += all_data[col].astype(str)+'_'
!echo "pneumonia" > cfg/rsna.names<install_modules>
cat_count_features = []
for col in cat_features+['mix_ind']:
val_counts_dic = all_data[col].value_counts().to_dict()
all_data[f'{col}_count'] = all_data[col].apply(lambda x: val_counts_dic[x])
 cat_count_features.append(f'{col}_count' )
!wget -q https://pjreddie.com/media/files/darknet53.conv.74<install_modules>
drop_features = ['ps_ind_14', 'ps_ind_10_bin','ps_ind_11_bin',
'ps_ind_12_bin','ps_ind_13_bin','ps_car_14']
all_data_remaining = all_data[remaining_features+cat_count_features].drop(drop_features, axis=1)
 all_data_sprs = sparse.hstack([sparse.csr_matrix(all_data_remaining),
encoded_cat_matrix],
format='csr' )
!wget --no-check-certificate -q "https://docs.google.com/uc?export=download&id=18ptTK4Vbeokqpux8Onr0OmwUP9ipmcYO" -O cfg/rsna_yolov3.cfg_train<define_variables>
num_train = train.shape[0]
X = all_data_sprs[:num_train]
X_test = all_data_sprs[num_train:]
y = train['target'].values
<install_modules>
def eval_gini(y_true, y_pred):
assert y_true.shape == y_pred.shape
n_samples = y_true.shape[0]
L_mid = np.linspace(1 / n_samples, 1, n_samples)
 pred_order = y_true[y_pred.argsort() ]
L_pred = np.cumsum(pred_order)/ np.sum(pred_order)
 G_pred = np.sum(L_mid - L_pred)
 true_order = y_true[y_true.argsort() ]
L_true = np.cumsum(true_order)/ np.sum(true_order)
 G_true = np.sum(L_mid - L_true)
 return G_pred / G_true
!wget --no-check-certificate -q "https://docs.google.com/uc?export=download&id=1OhnlV3s7r6xsEme6DKkNYjcYjsl-C_Av" -O train_log.txt<train_model>
def gini_lgb(preds, dtrain):
labels = dtrain.get_label()
return 'gini', eval_gini(labels, preds), True
ex_patient_id = annots[annots.Target == 1].patientId.values[2]
shutil.copy(ex_img_path, "test.jpg")
 print(ex_patient_id )<load_pretrained>
def gini_xgb(preds, dtrain):
labels = dtrain.get_label()
return 'gini', eval_gini(labels, preds )
!wget --load-cookies /tmp/cookies.txt -q "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1FDzMN-kGVYCvBeDKwemAazldSVkAEFyd' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+ ).*/\1
/p')&id=1FDzMN-kGVYCvBeDKwemAazldSVkAEFyd" -O backup/rsna_yolov3_15300.weights && rm -rf /tmp/cookies.txt<install_modules>
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1991 )
!wget --no-check-certificate -q "https://docs.google.com/uc?export=download&id=10Yk6ZMAKGz5LeBbikciALy82aK3lX-57" -O cfg/rsna_yolov3.cfg_test<install_modules>
max_params_lgb = {'bagging_fraction': 0.8043696643500143,
'feature_fraction': 0.6829323879981047,
'lambda_l1': 0.9264555612104627,
'lambda_l2': 0.9774233689434216,
'min_child_samples': 10,
'min_child_weight': 125.68433948868649,
'num_leaves': 28,
'objective': 'binary',
'learning_rate': 0.01,
'bagging_freq': 1,
'verbosity': 0,
'random_state': 1991}
!cd darknet &&./darknet detector test.. /cfg/rsna.data.. /cfg/rsna_yolov3.cfg_test.. /backup/rsna_yolov3_15300.weights.. /test.jpg -thresh 0.005<install_modules>
oof_val_preds_lgb = np.zeros(X.shape[0])
 oof_test_preds_lgb = np.zeros(X_test.shape[0])
 for idx,(train_idx, valid_idx)in enumerate(folds.split(X, y)) :
print('
X_train, y_train = X[train_idx], y[train_idx]
X_valid, y_valid = X[valid_idx], y[valid_idx]
dtrain = lgbm.Dataset(X_train, y_train)
 dvalid = lgbm.Dataset(X_valid, y_valid)
 lgb_model = lgbm.train(params=max_params_lgb,
train_set=dtrain,
num_boost_round=1500,
valid_sets=dvalid,
feval=gini_lgb,
early_stopping_rounds=150,
verbose_eval=100)
 best_iter = lgb_model.best_iteration
oof_test_preds_lgb += lgb_model.predict(X_test,
num_iteration=best_iter)/folds.n_splits
oof_val_preds_lgb[valid_idx] += lgb_model.predict(X_valid, num_iteration=best_iter)
 gini_score = eval_gini(y_valid, oof_val_preds_lgb[valid_idx])
 print(f'Fold {idx+1} gini score: {gini_score}
' )
!wget --no-check-certificate -q "https://docs.google.com/uc?export=download&id=1-KTV7K9G1bl3SmnLnzmpkDyNt6tDmH7j" -O darknet.py<import_modules>
max_params_xgb = {'colsample_bytree': 0.8927325521002059,
'gamma': 9.766883037651555,
'max_depth': 7,
'min_child_weight': 6.0577898395058085,
'reg_alpha': 8.136089122187865,
'reg_lambda': 1.385119327658532,
'scale_pos_weight': 1.5142072116395773,
'subsample': 0.717425859940308,
'objective': 'binary:logistic',
'learning_rate': 0.05,
'random_state': 1991}
from darknet import *<define_variables>
oof_val_preds_xgb = np.zeros(X.shape[0])
 oof_test_preds_xgb = np.zeros(X_test.shape[0])
 for idx,(train_idx, valid_idx)in enumerate(folds.split(X, y)) :
print('
X_train, y_train = X[train_idx], y[train_idx]
X_valid, y_valid = X[valid_idx], y[valid_idx]
dtrain = xgb.DMatrix(X_train, y_train)
 dvalid = xgb.DMatrix(X_valid, y_valid)
 dtest = xgb.DMatrix(X_test)
 watchlist = [(dtrain, 'train'),(dvalid, 'valid')]
xgb_model = xgb.train(params=max_params_xgb,
dtrain=dtrain,
num_boost_round=1000,
evals=watchlist,
maximize=True,
feval=gini_xgb,
early_stopping_rounds=150,
verbose_eval=100)
 best_iter = xgb_model.best_iteration
oof_test_preds_xgb += xgb_model.predict(dtest,
ntree_limit=best_iter)/folds.n_splits
oof_val_preds_xgb[valid_idx] += xgb_model.predict(dvalid, ntree_limit=best_iter)
 gini_score = eval_gini(y_valid, oof_val_preds_xgb[valid_idx])
 print(f'Fold {idx+1} gini score: {gini_score}
' )
threshold = 0.2<define_variables>
print('LightGBM OOF Gini Score:', eval_gini(y, oof_val_preds_lgb))
submit_file_path = "submission.csv"
cfg_path = os.path.join(cfg_dir, "rsna_yolov3.cfg_test")
 weight_path = os.path.join(backup_dir, "rsna_yolov3_15300.weights")
 test_img_list_path = os.path.join(metadata_dir, "te_list.txt" )<load_pretrained>
print('XGBoost OOF Gini Score:', eval_gini(y, oof_val_preds_xgb))
<categorify><EOS> 
oof_test_preds = oof_test_preds_lgb * 0.6 + oof_test_preds_xgb * 0.4
submission['target'] = oof_test_preds
submission.to_csv('submission.csv' )