Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<data_type_conversions>
import numpy as np
import pandas as pd
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
from sklearn.ensemble import VotingClassifier
col = 'grade'
mapping_dict = {
"grade": {
"A": 0,"B": 1,"C": 2,"D": 3,"E": 4,"F": 5,"G": 6
}
}
X_train = X_train.replace(mapping_dict)
 X_test = X_test.replace(mapping_dict)
 mapping_col = ['grade']
X_train[mapping_col] = X_train[mapping_col].fillna(-1)
 X_test[mapping_col] = X_test[mapping_col].fillna(-1)
 X_train[mapping_col] = X_train[mapping_col].astype(int)
 X_test[mapping_col] = X_test[mapping_col].astype(int)
 plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()<define_variables>
train_data = pd.read_csv('.. /input/titanic/train.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv' )
col = 'sub_grade'
mapping_dict = {
"sub_grade": {
"A1": 0,"A2": 1,"A3": 2,"A4": 3,"A5": 4,
"B1": 5,"B2": 6,"B3": 7,"B4": 8,"B5": 9,
"C1": 10,"C2": 11,"C3": 12,"C4": 13,"C5": 14,
"D1": 15,"D2": 16,"D3": 17,"D4": 18,"D5": 19,
"E1": 20,"E2": 21,"E3": 22,"E4": 23,"E5": 24,
"F1": 25,"F2": 26,"F3": 27,"F4": 28,"F5": 29,
"G1": 30,"G2": 31,"G3": 32,"G4": 33,"G5": 34
}
}
X_train = X_train.replace(mapping_dict)
 X_test = X_test.replace(mapping_dict)
 mapping_col = ['sub_grade']
X_train[mapping_col] = X_train[mapping_col].fillna(-1)
 X_test[mapping_col] = X_test[mapping_col].fillna(-1)
 X_train[mapping_col] = X_train[mapping_col].astype(int)
 X_test[mapping_col] = X_test[mapping_col].astype(int)
 plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show() 
<split>
missing_values = train_data.isna().any()
print('Columns which have missing values:
{0}'.format(missing_values[missing_values == True].index.tolist()))
col = 'emp_length'
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()<split>
print("Percentage of missing values in `Age` column: {0:.2f}".format(100.*(train_data.Age.isna().sum() /len(train_data))))
print("Percentage of missing values in `Cabin` column: {0:.2f}".format(100.*(train_data.Cabin.isna().sum() /len(train_data))))
print("Percentage of missing values in `Embarked` column: {0:.2f}".format(100.*(train_data.Embarked.isna().sum() /len(train_data))))
col = 'home_ownership'
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()<split>
duplicates = train_data.duplicated().sum()
print('Duplicates in train data: {0}'.format(duplicates))
col = 'purpose'
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()<split>
categorical = train_data.nunique().sort_values(ascending=True)
 print('Categorical variables in train data:
{0}'.format(categorical))
col = 'title'
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()<categorify>
for data in [train_data, test_data]:
data.drop(['Cabin'], axis=1, inplace=True)
 data.drop(['Ticket', 'Fare'], axis=1, inplace=True )
col = 'zip_code'
target = 'loan_condition'
X_train[col] = X_train[col].str[0:2]
X_test[col] = X_test[col].str[0:2]
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
max_threshold = 0.3
min_threshold = 0.1
X_train[col] = capping(X_train[col], min_threshold , max_threshold)
 X_test[col]= capping(X_test[col], min_threshold , max_threshold)
 plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show() 
<split>
for data in [train_data, test_data]:
data['Title'] = data.Name.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
data['Woman_Or_Boy'] =(data.Title == 'Master')|(data.Sex == 'female')
 data.drop('Title', axis=1, inplace=True)
 data.drop('Name', axis=1, inplace=True )
col = 'addr_state'
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
max_threshold = 0.25
min_threshold = 0.12
X_train[col] = capping(X_train[col], min_threshold , max_threshold)
 X_test[col]= capping(X_test[col], min_threshold , max_threshold)
 plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()<split>
label_encoder = LabelEncoder()
for data in [train_data, test_data]:
data['Sex'] = label_encoder.fit_transform(data['Sex'])
 data['Woman_Or_Boy'] = label_encoder.fit_transform(data['Woman_Or_Boy'] )
col = 'emp_title'
col4 = 'emp_title4'
target = 'loan_condition'
X_train[col4] = X_train[col].str[0:4]
X_test[col4] = X_test[col].str[0:4]
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col4])[target].mean()
X_test[col4] = X_test[col4].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col4])[target].mean()
enc_train.iloc[val_ix] = X_val[col4].map(summary)
 X_train[col4] = enc_train
max_threshold = 0.5
min_threshold = 0
X_train[col4] = capping(X_train[col4],min_threshold, max_threshold)
 X_test[col4]= capping(X_test[col4],min_threshold, max_threshold)
 plt.figure(figsize=[7,7])
 X_train[col4].hist(density=True, alpha=0.5, bins=20)
 X_test[col4].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col4)
 plt.ylabel('density')
 plt.show()
col = 'emp_title'
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
max_threshold = 0.5
min_threshold = 0
X_train[col] = capping(X_train[col],min_threshold, max_threshold)
 X_test[col]= capping(X_test[col],min_threshold, max_threshold)
 plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()<drop_column>
most_common = all_data.Embarked.mode()
print("Most common Embarked value: {0}".format(most_common[0]))
for data in [train_data, test_data]:
data.fillna(value={'Embarked': most_common[0]}, inplace=True )
drop_col = ['title','acc_now_delinq','grade','installment']
X_train = X_train.drop(columns=drop_col)
 X_test = X_test.drop(columns=drop_col )<define_variables>
for data in [train_data, test_data]:
data['TravelAlone'] = np.where(data["SibSp"] + data["Parch"] > 0, 0, 1)
 data.drop('SibSp', axis=1, inplace=True)
 data.drop('Parch', axis=1, inplace=True )
X_train1 = X_train
X_train2 = X_train
X_train3 = X_train
X_test1 = X_test
X_test2 = X_test
X_test3 = X_test<drop_column>
one_hot_encoder = OneHotEncoder(sparse=False)
 def encode_embarked(data):
encoded = pd.DataFrame(one_hot_encoder.fit_transform(data[['Embarked']]))
encoded.columns = one_hot_encoder.get_feature_names(['Embarked'])
 data.drop(['Embarked'], axis=1, inplace=True)
 data = data.join(encoded)
 return data
train_data = encode_embarked(train_data)
 test_data = encode_embarked(test_data )
drop_col = ['emp_title']
X_train1 = X_train1.drop(columns=drop_col)
 X_test1 = X_test1.drop(columns=drop_col )<drop_column>
X = train_data.drop(['Survived', 'PassengerId'], axis=1)
 y = train_data['Survived']
test_X = test_data.drop(['PassengerId'], axis=1 )
drop_col = ['emp_title']
X_train2 = X_train2.drop(columns=drop_col)
 X_test2 = X_test2.drop(columns=drop_col )<normalization>
best_models = {}
train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1)
 def print_best_parameters(hyperparameters, best_parameters):
value = "Best parameters: "
for key in hyperparameters:
value += str(key)+ ": " + str(best_parameters[key])+ ", "
if hyperparameters:
print(value[:-2])
 def get_best_model(estimator, hyperparameters):
cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
 grid_search = GridSearchCV(estimator=estimator, param_grid=hyperparameters,
n_jobs=-1, cv=cv, scoring="accuracy")
 best_model = grid_search.fit(train_X, train_y)
 best_parameters = best_model.best_estimator_.get_params()
print_best_parameters(hyperparameters, best_parameters)
 return best_model
def evaluate_model(model, name):
print("Accuracy score:", accuracy_score(train_y, model.predict(train_X)))
 best_models[name] = model
ratio_PCA = 0.95
sequence_col = ['sub_grade'
,'loan_amnt'
,'annual_inc'
,'dti'
,'Asset']
X_train_sequence = X_train2[sequence_col]
X_test_sequence = X_test2[sequence_col]
scaler = StandardScaler()
scaler.fit(X_train_sequence)
 X_train_sequence[sequence_col] = scaler.transform(X_train_sequence[sequence_col])
 X_test_sequence[sequence_col] = scaler.transform(X_test_sequence[sequence_col])
 X_train_sequence.fillna(X_train_sequence.median() , inplace=True)
 X_test_sequence.fillna(X_test_sequence.median() , inplace=True)
 pca = PCA(n_components=X_train_sequence.shape[1])
 pca.fit(X_train_sequence)
 sorted_variance = sorted(pca.explained_variance_ratio_,reverse=True)
 cum_sum = np.cumsum(sorted_variance)
 ratio = cum_sum / np.sum(sorted_variance)
 col_num = len(ratio)- len(ratio[ratio>ratio_PCA])+1
if col_num >= X_train_sequence.shape[1]:
col_num = X_train_sequence.shape[1]-1
sorted_variance = sorted(pca.explained_variance_ratio_,reverse=True)
 np.set_printoptions(suppress=True)
 svd = TruncatedSVD(n_components=col_num, n_iter=7, random_state=42)
 svd.fit(X_train_sequence)
 X_train_sequence = svd.transform(X_train_sequence)
 X_test_sequence = svd.transform(X_test_sequence)
 column_name = []
for c in range(0,col_num):
column_name.append('PCA' + str(c))
X_train_sequence = pd.DataFrame(X_train_sequence,columns = column_name)
 X_test_sequence = pd.DataFrame(X_test_sequence,columns = column_name)
 X_train_sequence.index = X_train.index
X_test_sequence.index = X_test.index
drop_col = sequence_col
X_train2 = X_train2.drop(columns=drop_col)
 X_test2 = X_test2.drop(columns=drop_col)
 X_train2 = pd.concat([X_train2, X_train_sequence], axis=1)
 X_test2 = pd.concat([X_test2, X_test_sequence], axis=1 )<drop_column>
class MyXGBClassifier(XGBClassifier):
def fit(self, X, y=None):
return super(XGBClassifier, self ).fit(X, y,
verbose=False,
early_stopping_rounds=40,
eval_metric='logloss',
eval_set=[(val_X, val_y)] )
drop_col = ['emp_title4']
X_train3 = X_train3.drop(columns=drop_col)
 X_test3 = X_test3.drop(columns=drop_col )<train_model>
randomForest = RandomForestClassifier(random_state=1, n_estimators=20, max_features='auto',
criterion='gini', max_depth=4, min_samples_split=2,
min_samples_leaf=3)
 xgbClassifier = MyXGBClassifier(seed=1, tree_method='gpu_hist', predictor='gpu_predictor',
use_label_encoder=False, learning_rate=0.4, gamma=0.4,
max_depth=4, reg_lambda=0, reg_alpha=0.1)
 lgbmClassifier = LGBMClassifier(random_state=1, device='gpu', boosting_type='dart',
num_leaves=8, learning_rate=0.1, n_estimators=100,
reg_alpha=1, reg_lambda=1)
 classifiers = [ 
('randomForest', randomForest), 
('xgbClassifier', xgbClassifier), 
('lgbmClassifier', lgbmClassifier)
 ]
clf = LGBMClassifier(boosting_type = 'gbdt',class_weight='balanced')
 clf.fit(X_train1, y_train, eval_metric='auc')
 y_pred1 = clf.predict_proba(X_test1)[:,1]
clf = LGBMClassifier(boosting_type = 'gbdt',class_weight='balanced')
 clf.fit(X_train2, y_train, eval_metric='auc')
 y_pred2 = clf.predict_proba(X_test2)[:,1]
clf = LGBMClassifier(boosting_type = 'gbdt',class_weight='balanced')
 clf.fit(X_train3, y_train, eval_metric='auc')
 y_pred3 = clf.predict_proba(X_test3)[:,1]
y_pred =(y_pred1+y_pred2+y_pred3)/3<save_to_csv>
hyperparameters = {
'n_jobs' : [-1],
'voting' : ['hard', 'soft'],
'weights' : [(1, 1, 1), 
(2, 1, 1),(1, 2, 1),(1, 1, 2), 
(2, 2, 1),(1, 2, 2),(2, 1, 2), 
(3, 2, 1),(1, 3, 2),(2, 1, 3),(3, 1, 2)]
}
estimator = VotingClassifier(estimators=classifiers)
 best_model_voting = get_best_model(estimator, hyperparameters )
submission = pd.read_csv('/kaggle/input/homework-for-students3/sample_submission.csv', index_col=0)
 submission.loan_condition = y_pred
submission.to_csv('submission.csv' )<import_modules>
evaluate_model(best_model_voting.best_estimator_, 'voting' )
import numpy as np
import pandas as pd
import seaborn as sb
import matplotlib.pyplot as plt
import sklearn as sk
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.pipeline import Pipeline
from sklearn.metrics import accuracy_score, roc_auc_score
from sklearn.model_selection import cross_validate<load_from_csv>
for model in best_models:
predictions = best_models[model].predict(test_X)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('submission_' + model + '.csv', index=False )