Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<init_hyperparams>
pd.plotting.register_matplotlib_converters()
%matplotlib inline
print('Setup complete')
params = {'num_leaves': int(LGB_BO.max['params']['num_leaves']),
'min_child_weight': LGB_BO.max['params']['min_child_weight'],
'feature_fraction': LGB_BO.max['params']['feature_fraction'],
'bagging_fraction': LGB_BO.max['params']['bagging_fraction'],
'min_data_in_leaf': int(LGB_BO.max['params']['min_data_in_leaf']),
'objective': 'binary',
'max_depth': int(LGB_BO.max['params']['max_depth']),
'learning_rate': 0.006883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
'seed': 11,
'drop_seed': 11,
'feature_fraction_seed': 11,
'data_random_seed': 11,
'metric': 'auc',
'verbosity': 1,
'reg_alpha': LGB_BO.max['params']['reg_alpha'],
'reg_lambda': LGB_BO.max['params']['reg_lambda'],
'random_state': 47
}<prepare_x_and_y>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 train.head()
NFOLDS = 5
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds = np.zeros(X_test.shape[0])
 y_oof = np.zeros(X.shape[0])
 score = 0
for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200, early_stopping_rounds=500)
 y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clf.predict(X_test)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
print(f"
Mean AUC = {score}")
 print(f"Out of folds AUC = {roc_auc_score(y, y_oof)}" )<save_to_csv>
train.isnull()
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['isFraud'] = y_preds
sub.to_csv('submission.csv', index=False )<set_options>
train.isna().sum()
seed = 10
np.random.seed(seed)
 random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
<load_from_csv>
train['Survived'].value_counts(normalize=True)
%%time
train_transaction = pd.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv', index_col='TransactionID' )<feature_engineering>
train.groupby('Pclass' ).mean() ['Survived']*100
train_transaction['hour'] = train_transaction['TransactionDT'].map(lambda x:(x//3600)%24)
 test_transaction['hour'] = test_transaction['TransactionDT'].map(lambda x:(x//3600)%24)
 train_transaction['weekday'] = train_transaction['TransactionDT'].map(lambda x:(x//(3600 * 24)) %7)
 test_transaction['weekday'] = test_transaction['TransactionDT'].map(lambda x:(x//(3600 * 24)) %7 )<feature_engineering>
train.groupby(['Pclass', 'Sex'] ).Survived.mean()
cols = "TransactionDT,TransactionAmt,ProductCD,card1,card2,card3,card4,card5,card6,addr1,addr2,C1,C2,C3,C4,C5,C6,C7,C8,C9,C10,C11,C12,C13,C14,M1,M2,M3,M4,M5,M6,M7,M8,M9".split(",")
 train_test = train_transaction[cols].append(test_transaction[cols])
 for col in "ProductCD,card1,card2,card3,card4,card5,card6,addr1,addr2,C1,C2,C3,C4,C5,C6,C7,C8,C9,C10,C11,C12,C13,C14".split(","):
col_count = train_test.groupby(col)['TransactionDT'].count()
train_transaction[col+'_count'] = train_transaction[col].map(col_count)
 test_transaction[col+'_count'] = test_transaction[col].map(col_count)
 for col in "card1,card2,card5,addr1,addr2".split(","):
col_count = train_test.groupby(col)['TransactionAmt'].mean()
train_transaction[col+'_amtcount'] = train_transaction[col].map(col_count)
 test_transaction[col+'_amtcount'] = test_transaction[col].map(col_count)
 col_count1 = train_test[train_test['C5'] == 0].groupby(col)['C5'].count()
col_count2 = train_test[train_test['C5'] != 0].groupby(col)['C5'].count()
train_transaction[col+'_C5count'] = train_transaction[col].map(col_count2)/(train_transaction[col].map(col_count1)+ 0.01)
 test_transaction[col+'_C5count'] = test_transaction[col].map(col_count2)/(test_transaction[col].map(col_count1)+ 0.01)
 del train_test 
<load_from_csv>
test['Titles'] = test['Name'].apply(lambda x: x.split(',')[1].split('.')[0].strip())
 train['Titles'] = train['Name'].apply(lambda x: x.split(',')[1].split('.')[0].strip())
 test['Titles'].value_counts()
train_identity = pd.read_csv('.. /input/ieee-fraud-detection/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/ieee-fraud-detection/test_identity.csv', index_col='TransactionID')
<drop_column>
train['Titles'].value_counts()
col_del = []
for i in range(339):
col = "V" + str(i+1)
 s = train_transaction[col].fillna(0 ).map(lambda x:0 if x%1 == 0 else 1 ).sum()
if s > 100:
print(col,s)
 col_del.append(col)
<categorify>
train['Titles'].replace(['Mme', 'Ms', 'Lady', 'Mlle', 'the Countess', 'Dona'], 'Miss', inplace=True)
 test['Titles'].replace(['Mme', 'Ms', 'Lady', 'Mlle', 'the Countess', 'Dona'], 'Miss', inplace=True)
 train['Titles'].replace(['Major', 'Col', 'Capt', 'Don', 'Sir', 'Jonkheer'], 'Mr', inplace=True)
 test['Titles'].replace(['Major', 'Col', 'Capt', 'Don', 'Sir', 'Jonkheer'], 'Mr', inplace=True)
 train.groupby('Titles' ).Survived.mean()
train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print(train.shape)
 print(test.shape)
 y_train = train['isFraud'].copy()
del train_transaction, train_identity, test_transaction, test_identity
X_train = train.drop('isFraud', axis=1)
 X_test = test.copy()
del train, test
for f in X_train.columns:
if X_train[f].dtype=='object' or X_test[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(X_train[f].values)+ list(X_test[f].values))
X_train[f] = lbl.transform(list(X_train[f].values))
X_test[f] = lbl.transform(list(X_test[f].values))<split>
train['Ticket_letters'] = train.Ticket.apply(lambda x: x[:2])
 test['Ticket_letters'] = test.Ticket.apply(lambda x: x[:2])
 train['Ticket_length'] = train.Ticket.apply(lambda x: len(x))
test['Ticket_length'] = test.Ticket.apply(lambda x: len(x))
%%time
X_train = reduce_mem_usage(X_train)
 X_test = reduce_mem_usage(X_test)
 debug = False
if debug:
split_pos = X_train.shape[0]*4//5
y_test = y_train.iloc[split_pos:]
y_train = y_train.iloc[:split_pos]
X_test = X_train.iloc[split_pos:,:]
X_train = X_train.iloc[:split_pos,:]<set_options>
y = train['Survived']
features = ['Pclass', 'Fare', 'Titles', 'Embarked', 'Fam_group', 'Ticket_length', 'Ticket_letters']
X = train[features]
X.head()
gc.collect()<train_model>
numerical_cols = ['Fare']
categorical_cols = ['Pclass', 'Titles', 'Embarked', 'Fam_group', 'Ticket_length', 'Ticket_letters']
numerical_transformer = SimpleImputer(strategy='median')
 categorical_transformer = Pipeline(steps=[('imputer', SimpleImputer(strategy='most_frequent')) ,('onehot', OneHotEncoder(handle_unknown='ignore')) ])
 preprocessor = ColumnTransformer(
 transformers=[('num', numerical_transformer, numerical_cols),('cat', categorical_transformer, categorical_cols)])
 titanic_pipeline = Pipeline(steps=[('preprocessor', preprocessor),('model', RandomForestClassifier(random_state=0, n_estimators=500, max_depth=5)) ])
 titanic_pipeline.fit(X,y)
 print('Cross validation score: {:.3f}'.format(cross_val_score(titanic_pipeline, X, y, cv=10 ).mean()))
%%time
folds = 3
kf = KFold(n_splits = folds, shuffle = True, random_state=seed)
 y_preds = np.zeros(X_test.shape[0])
 i = 0
for tr_idx, val_idx in kf.split(X_train, y_train):
i+=1
clf = xgb.XGBClassifier(
 n_estimators=700,
max_depth=9,
learning_rate=0.03,
subsample=0.9,
colsample_bytree=0.9,
tree_method='gpu_hist' 
)
 X_tr = X_train.iloc[tr_idx, :]
y_tr = y_train.iloc[tr_idx]
clf.fit(X_tr, y_tr)
 del X_tr
y_preds+= clf.predict_proba(X_test)[:,1] / folds
if debug:
print("debug:",roc_auc_score(y_test, clf.predict_proba(X_test)[:,1] / folds))
del clf
if debug:
print("debug:",roc_auc_score(y_test, y_preds))
gc.collect()<split>
predictions = titanic_pipeline.predict(X_test )
<init_hyperparams><EOS> 
o = pd.DataFrame({'PassengerId': test.PassengerId, 'Survived': predictions})
 o.to_csv('my_submission.csv', index=False)
 print('Your submission was successfully saved!' )