Kaggle competition: Titanic - Machine Learning from Disaster
def add_new_feature_row(df,features):
for feature in features:
df[feature+"_pct"] = df[feature].pct_change()
df[feature+"_diff"] = df[feature].diff()
df.drop(feature,axis=1)
 return df 
<feature_engineering>
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 train_data.head()
def normalize_df(df,features):
for feature in features:
df[feature+'_norm'] =(df[feature] - df[feature].mean())/df[feature].std()
return df 
<sort_values>
test_data = pd.read_csv('/kaggle/input/titanic/test.csv')
 test_data.head()
correlations = train_df[features].corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']] 
<set_options>
categorical_cols = [col for col in train_data.columns if train_data[col].dtype == object
and train_data[col].nunique() <= 10]
print("Categorical columns: ",categorical_cols)
 categorical_cols_missing_val = [col for col in train_data[categorical_cols].columns
if train_data[col].isnull().any() ]
print('
Categorical columns with missing values: ')
 for col in categorical_cols_missing_val:
print(col,'-', train_data[col].isnull().sum() ,'missing value(s)')
 print('
Shape of "train_data":',train_data.shape )
gc.collect()<concatenate>
categorical_test_cols = [col for col in test_data.columns
if test_data[col].nunique() <= 10 and test_data[col].dtype == object]
print("Categorical columns: ",categorical_test_cols)
 categorical_test_cols_missing_val = [col for col in test_data[categorical_test_cols].columns
if test_data[col].isnull().any() ]
print('
Categorical columns with missing values: None')
 for col in categorical_test_cols_missing_val:
print(col,'-', test_data[col].isnull().sum() ,'missing value(s)')
 print('
Shape of "test_data":',test_data.shape )
test_df['target']= np.nan
combine_df = train_df.append(test_df,ignore_index=True )<concatenate>
numerical_cols = [col for col in train_data.columns if train_data[col].dtype != object]
numerical_cols.remove('Survived')
 numerical_cols.remove('PassengerId')
 print("Numerical columns: ",numerical_cols)
 num_cols_with_missing = [col for col in train_data[numerical_cols].columns
if train_data[col].isnull().any() ]
print('
Nummerical columns with missing values:')
 for col in num_cols_with_missing:
print(col,'-', train_data[col].isnull().sum() ,'missing value(s)')
 print('
Shape of "train_data"',train_data.shape )
features = train_df.columns.values[2:]
combine_df = normalize_df(combine_df,features )<split>
numerical_test_cols = [col for col in test_data.columns if test_data[col].dtype != object]
numerical_test_cols.remove('PassengerId')
 print("Numerical columns: ",numerical_test_cols)
 num_test_cols_with_missing = [col for col in test_data[numerical_test_cols].columns
if test_data[col].isnull().any() ]
print('
Nummerical columns with missing values:')
 for col in num_test_cols_with_missing:
print(col,'-', test_data[col].isnull().sum() ,'missing value(s)')
 print('
Shape of "test_data"',test_data.shape )
train_df = combine_df[combine_df['target'].notnull() ].reset_index(drop=True)
 test_df = combine_df[combine_df['target'].isnull() ].reset_index(drop=True)
<define_variables>
my_cols = numerical_cols + categorical_cols
predictors = train_df.columns.values.tolist() [2:]
nfold = 10
target = 'target'<init_hyperparams>
print('Number of unique values in X
',train_data[my_cols].nunique() ,'
' )
param = {
'num_leaves': 18,
'max_bin': 63,
'min_data_in_leaf': 5,
'learning_rate': 0.010614430970330217,
'min_sum_hessian_in_leaf': 0.0093586657313989123,
'feature_fraction': 0.056701788569420042,
'lambda_l1': 0.060222413158420585,
'lambda_l2': 4.6580550589317573,
'min_gain_to_split': 0.29588543202055562,
'max_depth': 49,
'save_binary': True,
'seed': 1337,
'feature_fraction_seed': 1337,
'bagging_seed': 1337,
'drop_seed': 1337,
'data_random_seed': 1337,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': False
}
nfold = 10
skf = StratifiedKFold(n_splits=nfold, shuffle=True, random_state=42)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
i = 1
for train_index, valid_index in skf.split(train_df, train_df.target.values):
print("
fold {}".format(i))
xg_train = lgb.Dataset(train_df.iloc[train_index][predictors].values,
label=train_df.iloc[train_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 xg_valid = lgb.Dataset(train_df.iloc[valid_index][predictors].values,
label=train_df.iloc[valid_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 nround = 8523
clf = lgb.train(param,
xg_train,
nround,
valid_sets = [xg_valid],
early_stopping_rounds=250,
verbose_eval=250)
 oof[valid_index] = clf.predict(train_df.iloc[valid_index][predictors].values, num_iteration=nround)
 predictions += clf.predict(test_df[predictors], num_iteration=nround)/ nfold
i = i + 1
print("
CV AUC: {:<0.4f}".format(metrics.roc_auc_score(train_df.target.values, oof)) )<save_to_csv>
women = train_data.loc[train_data.Sex == 'female']["Survived"]
rate_women = sum(women)/len(women)
 print("% of women who survived:", rate_women)
 men = train_data.loc[train_data.Sex == 'male']["Survived"]
rate_men = sum(men)/len(men)
 print("% of men who survived:", rate_men )
sub_df = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub_df["target"] = predictions
sub_df.to_csv("sant_lgb.csv", index=False)
 sub_df[:10]<set_options>
numerical_cols.remove('Age' )
warnings.filterwarnings('ignore')
<load_from_csv>
categorical_cols.remove('Embarked' )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )<prepare_x_and_y>
y = train_data.Survived
X = train_data.drop(['Survived','Name'], axis=1)
 train_X_full, val_X_full, train_y, val_y = train_test_split(X, y,
train_size=0.8, test_size=0.2,
random_state=0)
 train_X = train_X_full[my_cols].copy()
val_X = val_X_full[my_cols].copy()
train_cols = [c for c in train_df.columns if c not in ["ID_code", "target"]]
y_train = train_df["target"]<count_values>
numerical_transformer = SimpleImputer(strategy='median')
 categorical_transformer = Pipeline(steps=
[('imputer', SimpleImputer(strategy='constant')) , 
('onehot', OneHotEncoder(handle_unknown='ignore'))
])
 preprocessor = ColumnTransformer(
 transformers=[ 
('num', numerical_transformer, numerical_cols), 
('cat', categorical_transformer, categorical_cols)
 ])
 rf_model = RandomForestClassifier(n_estimators=72,
random_state=0,
max_leaf_nodes=50,
max_depth=10)
 pipe_rf = Pipeline(steps=
[('preprocessor', preprocessor), 
('model', rf_model)])
 pipe_rf.fit(train_X, train_y)
 preds_rf = pipe_rf.predict(val_X)
 mae_rf = mean_absolute_error(val_y, preds_rf)
 print("Validation MAE for Random Forest Classifier: %0.4f"%(mae_rf))
xgb_model = XGBClassifier(n_estimators=800,
learning_rate=0.01)
 preps = Pipeline(steps=
[('preprocessor', preprocessor), 
('xgbrg', xgb_model)])
 preps.fit(train_X, train_y)
 preds_xgb = preps.predict(val_X)
 mae_xgb = mean_absolute_error(preds_xgb, val_y)
 print("Validation MAE for XGBoost Classifier: %0.4f" %mae_xgb )
y_train.value_counts()<choose_model_class>
my_cols = numerical_cols + categorical_cols
my_model_full_data = RandomForestClassifier(n_estimators=72,
random_state=0,
max_leaf_nodes=50,
max_depth=10)
 prep_full_data = Pipeline(steps=
[('preprocessor', preprocessor), 
('model', my_model_full_data)])
 prep_full_data.fit(X[my_cols], y )
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1001 )<init_hyperparams>
test_X = test_data[my_cols].copy()
params = {'tree_method': 'hist',
'objective': 'binary:logistic',
'eval_metric': 'auc',
'learning_rate': 0.0936165921314771,
'max_depth': 2,
'colsample_bytree': 0.3561271102144279,
'subsample': 0.8246604621518232,
'min_child_weight': 53,
'gamma': 9.943467991283027,
'silent': 1}<split>
test_preds = prep_full_data.predict(test_X)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived':test_preds})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )