Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<find_best_params>
warnings.filterwarnings('ignore' )
def objective(trial):
params = {
'objective': 'binary',
'metric': 'binary_logloss',
'boosting_type': 'gbdt',
'is_unbalance': False,
'boost_from_average': True,
'num_threads': 4,
'num_leaves': trial.suggest_int('num_leaves', 10, 1000),
'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 10, 200),
'min_child_weight': trial.suggest_loguniform('min_child_weight', 0.001, 0.1),
'max_depth': trial.suggest_int('max_depth', 1, 100),
'bagging_fraction' : trial.suggest_loguniform('bagging_fraction',.5,.99),
'feature_fraction' : trial.suggest_loguniform('feature_fraction',.5,.99),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 0.1, 2),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 0.1, 2)
 }
scores = []
cv = KFold(n_splits=5)
 for train_idx, valid_idx in cv.split(x_train, y_train):
x_train_train = x_train.iloc[train_idx]
y_train_train = y_train.iloc[train_idx]
x_train_valid = x_train.iloc[valid_idx]
y_train_valid = y_train.iloc[valid_idx]
lgb_train = lgb.Dataset(data=x_train_train.astype('float32'), label=y_train_train.astype('float32'))
lgb_valid = lgb.Dataset(data=x_train_valid.astype('float32'), label=y_train_valid.astype('float32'))
lgb_model = lgb.train(params, lgb_train, valid_sets=lgb_valid, verbose_eval=100)
 y = lgb_model.predict(x_train_valid.astype('float32'), num_iteration=lgb_model.best_iteration)
 score = roc_auc_score(y_train_valid.astype('float32'), y)
 print('Fold score:', score)
 scores.append(score)
 average_score = sum(scores)/ len(scores)
 print('Average score:', average_score)
 return average_score 
<init_hyperparams>
raw_data = pd.read_csv(".. /input/train.csv")
 raw_test = pd.read_csv('.. /input/test.csv' )
params = {
'objective': 'binary',
'metric': 'auc',
'boosting_type': 'gbdt',
'is_unbalance': False,
'boost_from_average': True,
'num_threads': 4,
'num_iterations': 5000,
'learning_rate': 0.006,
'early_stopping_round': 100,
'num_leaves': 396,
'min_data_in_leaf': 39,
'min_child_weight': 0.06046322469906681,
'max_depth': 95,
'bagging_fraction' : 0.7119300452429695,
'feature_fraction' : 0.6121880522551549,
'lambda_l1': 0.30642889371923837,
'lambda_l2': 0.6028242575804484
}<data_type_conversions>
def survival_estimator(x):
return len(x[x==1])/len(x)*100.0
n_splits=5
y = np.zeros(x_test.shape[0])
 feature_importances = []
cv = KFold(n_splits=n_splits)
 for train_idx, valid_idx in cv.split(x_train, y_train):
x_train_train = x_train.iloc[train_idx]
y_train_train = y_train.iloc[train_idx]
x_train_valid = x_train.iloc[valid_idx]
y_train_valid = y_train.iloc[valid_idx]
lgb_train = lgb.Dataset(data=x_train_train.astype('float32'), label=y_train_train.astype('float32'))
lgb_valid = lgb.Dataset(data=x_train_valid.astype('float32'), label=y_train_valid.astype('float32'))
lgb_model = lgb.train(params, lgb_train, valid_sets=lgb_valid, verbose_eval=100, early_stopping_rounds=100)
 y_part = lgb_model.predict(x_test.astype('float32'), num_iteration=lgb_model.best_iteration)
 y += y_part / n_splits
feature_importances.append(lgb_model.feature_importance() )<save_to_csv>
print(raw_data.isnull().sum())
 print("-"*10)
 print(raw_data.isnull().sum() /raw_data.shape[0] )
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID')
 submission['isFraud'] = y
submission.to_csv('lightgbm.csv' )<load_from_csv>
raw_data['IsAgeNull'] = np.where(np.isnan(raw_data['Age']), 1, 0)
 raw_data['Age'].fillna(( raw_data['Age'].mean()), inplace=True)
 raw_data['Age'] = raw_data['Age'].round().astype(int )
%%time
train_transaction = pd.read_csv('.. /input/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print(train.shape)
 print(test.shape )<set_options>
raw_data['AgeLabel'] = pd.cut(raw_data['Age'], bins=np.arange(start=0, stop=90, step=10), labels=np.arange(start=0, stop=8, step=1), include_lowest=True )
def reduce_mem_usage(df):
start_mem_usg = df.memory_usage().sum() / 1024**2
print("Memory usage of properties dataframe is :",start_mem_usg," MB")
 NAlist = []
for col in df.columns:
if df[col].dtype != object:
print("******************************")
 print("Column: ",col)
 print("dtype before: ",df[col].dtype)
 IsInt = False
mx = df[col].max()
mn = df[col].min()
print("min for this col: ",mn)
 print("max for this col: ",mx)
 if not np.isfinite(df[col] ).all() :
NAlist.append(col)
 df[col].fillna(mn-1,inplace=True)
 asint = df[col].fillna(0 ).astype(np.int64)
 result =(df[col] - asint)
 result = result.sum()
if result > -0.01 and result < 0.01:
IsInt = True
if IsInt:
if mn >= 0:
if mx < 255:
df[col] = df[col].astype(np.uint8)
 elif mx < 65535:
df[col] = df[col].astype(np.uint16)
 elif mx < 4294967295:
df[col] = df[col].astype(np.uint32)
 else:
df[col] = df[col].astype(np.uint64)
 else:
if mn > np.iinfo(np.int8 ).min and mx < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif mn > np.iinfo(np.int16 ).min and mx < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif mn > np.iinfo(np.int32 ).min and mx < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif mn > np.iinfo(np.int64 ).min and mx < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
df[col] = df[col].astype(np.float32)
 print("dtype after: ",df[col].dtype)
 print("******************************")
 print("___MEMORY USAGE AFTER COMPLETION:___")
 mem_usg = df.memory_usage().sum() / 1024**2
print("Memory usage is: ",mem_usg," MB")
 print("This is ",100*mem_usg/start_mem_usg,"% of the initial size")
 return df, NAlist<prepare_x_and_y>
raw_data['FareLabel'] = pd.cut(raw_data.Fare, bins=np.arange(start=0, stop=600, step=50), precision=0, labels=np.arange(start=0, stop=11, step=1), include_lowest=True )
y_train = train['isFraud'].copy()
del train_transaction, train_identity, test_transaction, test_identity
X_train = train.drop('isFraud', axis=1)
 X_test = test.copy()
del train, test 
<categorify>
raw_data = pd.concat([raw_data, pd.get_dummies(raw_data['Sex'])], axis=1 )
X_train = X_train.fillna(-999)
 X_test = X_test.fillna(-999)
 for f in X_train.columns:
if X_train[f].dtype=='object' or X_test[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(X_train[f].values)+ list(X_test[f].values))
X_train[f] = lbl.transform(list(X_train[f].values))
X_test[f] = lbl.transform(list(X_test[f].values))<concatenate>
raw_data = pd.concat([raw_data, pd.get_dummies(raw_data['Title'], prefix='title')], axis=1 )
X=pd.concat([X_train,y_train],axis=1)
<count_values>
raw_data = pd.concat([raw_data, pd.get_dummies(raw_data['Pclass'], prefix='Pclass')], axis=1 )
X['isFraud'].value_counts()<randomize_order>
need_columns = [
'female', 'male',
'Pclass_1', 'Pclass_2', 'Pclass_3',
'title_Master', 'title_Miss', 'title_Mr', 'title_Mrs', 'title_Rare',
'AgeLabel', 'IsAgeNull', 'IsChildren',
'FareLabel', 'SibSp', 'Parch', 'FamilySize',
'NoFamily', 'SmallFamily', 'MediumFamily', 'LargeFamily',
'LowFare', 'HighFare', 'MediumFare',
'NoCabin'
]
data = raw_data[need_columns]
x = data
y = raw_data.Survived
not_fraud=X[X.isFraud==0]
fraud=X[X.isFraud==1]
fraud_upsampled = resample(fraud,
replace=True,
n_samples=len(not_fraud),
random_state=27)
 upsampled = pd.concat([not_fraud, fraud_upsampled])
 upsampled.isFraud.value_counts()<create_dataframe>
n_estimators = [int(x)for x in np.linspace(start = 100, stop = 2000, num = 20)]
max_features = ['auto', 'sqrt', 'log2', None]
max_depth = [int(x)for x in np.linspace(10, 200, num = 20)]
max_depth.append(None)
 min_samples_split = [2, 5, 10]
min_samples_leaf = [1, 2, 4]
bootstrap = [True, False]
max_leaf_nodes = [2, 5, 8, 10, None]
criterion=['gini', 'entropy']
random_grid = {
'n_estimators': n_estimators,
'criterion': criterion,
'max_features': max_features,
'max_depth': max_depth,
'min_samples_split': min_samples_split,
'min_samples_leaf': min_samples_leaf,
'bootstrap': bootstrap,
'max_leaf_nodes': max_leaf_nodes
}
estimator = RandomForestClassifier()
columns = []
corr =corr1[np.abs(corr1)> 0.3]
corr = pd.DataFrame(data=corr,columns= ['isFraud'])
 corr = corr.drop_duplicates().sort_values('isFraud',ascending = False)
 corr.shape<prepare_x_and_y>
rf_random = RandomizedSearchCV(
 estimator=estimator,
param_distributions=random_grid,
random_state=42,
n_jobs=-1 
)
 rf_random.fit(x, y )
X_train_1 = filter_data
X_train_1 = X_train_1.drop(columns = ['isFraud'] , axis =1)
 y_train_1 = filter_data['isFraud']<choose_model_class>
best_params = rf_random.best_params_
print(best_params )
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
missing=-999,
random_state=2019,
tree_method='gpu_hist' 
)
<train_model>
best_score = rf_random.best_score_
print("best score {}".format(best_score))
%time clf.fit(X_train_1, y_train_1 )<save_to_csv>
test_model = RandomForestClassifier(
 n_estimators=best_params['n_estimators'],
criterion=best_params['criterion'],
max_features=best_params['max_features'],
max_depth=best_params['max_depth'],
min_samples_split=best_params['min_samples_split'],
min_samples_leaf=best_params['min_samples_leaf'],
max_leaf_nodes=best_params['max_leaf_nodes'],
bootstrap=best_params['bootstrap']
)
sample_submission['isFraud'] = clf.predict_proba(X_test_1)[:,1]
sample_submission.to_csv('simple_xgboost_2.csv' )<string_transform>
scores = cross_val_score(test_model, x, y, cv=10, scoring='accuracy' )
n_fold = 5
folds = TimeSeriesSplit(n_splits=n_fold)
 folds = KFold(n_splits=5 )<train_model>
print(scores)
 print("Mean Accuracy: {}".format(scores.mean()))
def train_model_classification(X, X_test, y, params, folds, model_type='lgb', eval_metric='auc', columns=None, plot_feature_importance=False, model=None,
verbose=10000, early_stopping_rounds=200, n_estimators=50000, splits=None, n_folds=3, averaging='usual', n_jobs=-1):

 columns = X.columns if columns is None else columns
n_splits = folds.n_splits if splits is None else n_folds
X_test = X_test[columns]
metrics_dict = {'auc': {'lgb_metric_name': eval_auc,
'catboost_metric_name': 'AUC',
'sklearn_scoring_function': metrics.roc_auc_score},
}
result_dict = {}
if averaging == 'usual':
oof = np.zeros(( len(X), 1))
prediction = np.zeros(( len(X_test), 1))
elif averaging == 'rank':
oof = np.zeros(( len(X), 1))
prediction = np.zeros(( len(X_test), 1))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 if type(X)== np.ndarray:
X_train, X_valid = X[columns][train_index], X[columns][valid_index]
y_train, y_valid = y[train_index], y[valid_index]
else:
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
model = lgb.LGBMClassifier(**params, n_estimators=n_estimators, n_jobs = n_jobs)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric=metrics_dict[eval_metric]['lgb_metric_name'],
verbose=verbose, early_stopping_rounds=early_stopping_rounds)
 y_pred_valid = model.predict_proba(X_valid)[:, 1]
y_pred = model.predict_proba(X_test, num_iteration=model.best_iteration_)[:, 1]
if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=n_estimators, evals=watchlist, early_stopping_rounds=early_stopping_rounds, verbose_eval=verbose, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid)
 print(f'Fold {fold_n}.{eval_metric}: {score:.4f}.')
 print('')
 y_pred = model.predict_proba(X_test)
 if model_type == 'cat':
model = CatBoostClassifier(iterations=n_estimators, eval_metric=metrics_dict[eval_metric]['catboost_metric_name'], **params,
loss_function=metrics_dict[eval_metric]['catboost_metric_name'])
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 if averaging == 'usual':
oof[valid_index] = y_pred_valid.reshape(-1, 1)
 scores.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid))
prediction += y_pred.reshape(-1, 1)
 elif averaging == 'rank':
oof[valid_index] = y_pred_valid.reshape(-1, 1)
 scores.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid))
prediction += pd.Series(y_pred ).rank().values.reshape(-1, 1)
 if model_type == 'lgb' and plot_feature_importance:
fold_importance = pd.DataFrame()
fold_importance["feature"] = columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 result_dict['oof'] = oof
result_dict['prediction'] = prediction
result_dict['scores'] = scores
if model_type == 'lgb':
if plot_feature_importance:
feature_importance["importance"] /= n_splits
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
result_dict['feature_importance'] = feature_importance
result_dict['top_columns'] = cols
return result_dict<compute_test_metric>
model = RandomForestClassifier(
 n_estimators=best_params['n_estimators'],
criterion=best_params['criterion'],
max_features=best_params['max_features'],
max_depth=best_params['max_depth'],
min_samples_split=best_params['min_samples_split'],
min_samples_leaf=best_params['min_samples_leaf'],
max_leaf_nodes=best_params['max_leaf_nodes'],
bootstrap=best_params['bootstrap'] 
)
 model.fit(x, y )
def eval_auc(y_true, y_pred):

 return 'auc', fast_auc(y_true, y_pred), True 
<compute_test_metric>
print(raw_test.isnull().sum())
 print("-"*10)
 print(raw_test.isnull().sum() /raw_test.shape[0] )
def fast_auc(y_true, y_prob):

 y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
nfalse = 0
auc = 0
n = len(y_true)
 for i in range(n):
y_i = y_true[i]
nfalse +=(1 - y_i)
 auc += y_i * nfalse
auc /=(nfalse *(n - nfalse))
return auc 
<init_hyperparams>
raw_test['Fare'].fillna(raw_test['Fare'].mean() , inplace=True)
 raw_test['NoCabin'] = np.where(raw_test['Cabin'].isnull() , 1, 0)
 raw_test['IsChildren'] = np.where(raw_test['Age']<=10, 1, 0)
 raw_test['IsAgeNull'] = np.where(np.isnan(raw_test['Age']), 1, 0)
 raw_test['Age'].fillna(raw_test['Age'].mean() , inplace=True)
 raw_test['Age'] = raw_test['Age'].round().astype(int)
 raw_test['AgeLabel'] = pd.cut(raw_test['Age'], bins=np.arange(start=0, stop=90, step=10), labels=np.arange(start=0, stop=8, step=1), include_lowest=True)
 raw_test['FamilySize'] = raw_test.apply(lambda row: row['SibSp']+row['Parch'], axis=1)
 raw_test['LowFare'] = np.where(raw_test['Fare']<=50, 1, 0)
 raw_test['HighFare'] = np.where(raw_test['Fare']>300, 1, 0)
 raw_test['MediumFare'] = raw_test.apply(medium_fare, axis=1)
 raw_test['FareLabel'] = pd.cut(raw_test.Fare, bins=np.arange(start=0, stop=600, step=50), precision=0, labels=np.arange(start=0, stop=11, step=1), include_lowest=True)
 raw_test['NoFamily'] = np.where(raw_test['FamilySize']==0, 1, 0)
 raw_test['SmallFamily'] = np.where(( raw_test['FamilySize']>0)&(raw_test['FamilySize']<4), 1, 0)
 raw_test['MediumFamily'] = np.where(( raw_test['FamilySize']>3)&(raw_test['FamilySize']<7), 1, 0)
 raw_test['LargeFamily'] = np.where(raw_test['FamilySize']>=7, 1, 0)
 raw_test['Title'] = raw_test.Name.str.extract('([A-Za-z]+)\.', expand=False)
 raw_test['Title'] = raw_test['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 raw_test['Title'] = raw_test['Title'].replace('Mlle', 'Miss')
 raw_test['Title'] = raw_test['Title'].replace('Ms', 'Miss')
 raw_test['Title'] = raw_test['Title'].replace('Mme', 'Mrs')
 raw_test = pd.concat([raw_test, pd.get_dummies(raw_test['Sex'])], axis=1)
 raw_test = pd.concat([raw_test, pd.get_dummies(raw_test['Title'], prefix='title')], axis=1)
 raw_test = pd.concat([raw_test, pd.get_dummies(raw_test['Pclass'], prefix='Pclass')], axis=1)
 data_test = raw_test[need_columns]
<categorify><EOS> 
ids = raw_test['PassengerId']
predictions = model.predict(data_test)
 output = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 output.to_csv('submission.csv', index = False)
 output.head(10 )