Kaggle competition: Titanic - Machine Learning from Disaster
train = train.replace(np.inf,999)
 test = test.replace(np.inf,999 )<concatenate>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import RandomizedSearchCV
from collections import Counter
i_cols = ['card1','card2','card3','card5',
'C1','C2','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14',
'D1','D2','D3','D4','D5','D6','D7','D8',
'addr1','addr2',
'dist1','dist2',
'P_emaildomain', 'R_emaildomain',
'DeviceInfo','device_name',
'id_30','id_33',
'uid','uid2','uid3',
]
for col in i_cols:
temp_df = pd.concat([train[[col]], test[[col]]])
 fq_encode = temp_df[col].value_counts(dropna=False ).to_dict()
train[col+'_fq_enc'] = train[col].map(fq_encode)
 test[col+'_fq_enc'] = test[col].map(fq_encode)
 for col in ['DT_M','DT_W','DT_D']:
temp_df = pd.concat([train[[col]], test[[col]]])
 fq_encode = temp_df[col].value_counts().to_dict()
train[col+'_total'] = train[col].map(fq_encode)
 test[col+'_total'] = test[col].map(fq_encode)
 periods = ['DT_M','DT_W','DT_D']
i_cols = ['uid']
for period in periods:
for col in i_cols:
new_column = col + '_' + period
temp_df = pd.concat([train[[col,period]], test[[col,period]]])
 temp_df[new_column] = temp_df[col].astype(str)+ '_' +(temp_df[period] ).astype(str)
 fq_encode = temp_df[new_column].value_counts().to_dict()
train[new_column] =(train[col].astype(str)+ '_' + train[period].astype(str)).map(fq_encode)
 test[new_column] =(test[col].astype(str)+ '_' + test[period].astype(str)).map(fq_encode)
 train[new_column] /= train[period+'_total']
test[new_column] /= test[period+'_total']<drop_column>
train=pd.read_csv('/kaggle/input/titanic/train.csv')
 test=pd.read_csv('/kaggle/input/titanic/test.csv')
 target=train['Survived']
def detect_outlier(df,n,cols):
outlier_indices = []
for i in cols:
Q1 = np.percentile(df[i], 20)
 Q3 = np.percentile(df[i], 80)
 IQR = Q3 - Q1
outlier_step = 1.5*IQR
outlier_index_list = df[(df[i] < Q1-outlier_step)|(df[i] > Q3+outlier_step)].index
outlier_indices.extend(outlier_index_list)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k,v in outlier_indices.items() if v>n)
 return multiple_outliers
outliers_to_drop = detect_outlier(train,3,['Age', 'SibSp', 'Parch', 'Fare'])
 train = train.drop(outliers_to_drop, axis = 0 ).reset_index(drop=True)
 print('Shape of train dataset:-',train.shape)
 print('Shape of test dataset:-' ,test.shape)
 print('
')
 print(train.info())
 train.describe()
train.drop(['TransactionDT', 'uid','uid2','uid3', 'DT','DT_M','DT_W','DT_D', 'DT_hour','DT_day_week','DT_day',
'DT_D_total','DT_W_total','DT_M_total', 'id_30','id_31','id_33', 'D1', 'D2', 'D9'], axis = 1, inplace = True)
 test.drop(['TransactionDT', 'uid','uid2','uid3', 'DT','DT_M','DT_W','DT_D', 'DT_hour','DT_day_week','DT_day',
'DT_D_total','DT_W_total','DT_M_total', 'id_30','id_31','id_33', 'D1', 'D2', 'D9'], axis = 1, inplace = True )<data_type_conversions>
total=pd.concat([train.drop('Survived',axis=1),test])
 target=train['Survived']
total.head()
for col in train.columns:
if train[col].dtype == 'object':
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))<feature_engineering>
print(total.isnull().sum())
 total['Age'] = total.groupby('Pclass')['Age'].transform(lambda x: x.fillna(x.median()))
total['Fare'] = total.groupby('Pclass')['Fare'].transform(lambda x: x.fillna(x.median()))
total['Embarked'].fillna('S',inplace=True)
def agg_features(df):
columns_a = ['TransactionAmt', 'id_02', 'D15']
columns_b = ['card1', 'card4', 'addr1']
for col_a in columns_a:
for col_b in columns_b:
df[f'{col_a}_to_mean_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('mean')
 df[f'{col_a}_to_std_{col_b}'] = df[col_a] / df.groupby([col_b])[col_a].transform('std')
 return df
test = agg_features(test)
 train = agg_features(train )<drop_column>
encoder=LabelEncoder()
total['Sex']=encoder.fit_transform(total['Sex'])
 total['Embarked']=encoder.fit_transform(total['Embarked'])
 total=pd.get_dummies(total,columns=['Pclass','Embarked'] )
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )<prepare_x_and_y>
total['Fare_1_S']=total['Embarked_2']*total['Pclass_1']*total['Sex']
X = train.drop(['isFraud'], axis = 1)
 y = train['isFraud']
print('Our train set have {} columns'.format(train.shape[1]))
print('Our test set have {} columns'.format(test.shape[1]))
gc.collect()<init_hyperparams>
total['Title'] =total['Name'].str.extract('([A-Za-z]+)\.', expand=False)
 total['Title'] =total['Title'].replace(['Lady', 'Countess','Capt', 'Col', 'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 total['Title'] =total['Title'].replace('Mlle', 'Miss')
 total['Title'] =total['Title'].replace('Ms', 'Miss')
 total['Title'] =total['Title'].replace('Mme', 'Mrs')
 title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 2, "Master": 3, "Rare": 4}
total['Title'] =total['Title'].map(title_mapping)
 total['Title'] =total['Title'].fillna(0)
 total['Age_cat'] = pd.qcut(total['Age'],q=[0,.16,.33,.49,.66,.83, 1], labels=False, precision=1)
 def fare_category(fr):
if fr <= 7.91:
return 1
elif fr <= 14.454 and fr > 7.91:
return 2
elif fr <= 31 and fr > 14.454:
return 3
return 4
total['Fare_cat'] =total['Fare'].apply(fare_category)
 total['FamilySize'] =total['SibSp'] + total['Parch'] + 1
total['FamilySize_cat'] =total['FamilySize'].map(lambda x: 1 if x == 1
else(2 if 5 > x >= 2
else(3 if 8 > x >= 5
else 4)
))
params = {
'objective':'binary',
'boosting_type':'gbdt',
'metric':'auc',
'n_jobs':-1,
'learning_rate':0.005,
'num_leaves': 2**8,
'max_depth':-1,
'tree_learner':'serial',
'colsample_bytree': 0.7,
'subsample_freq':1,
'subsample':0.7,
'n_estimators':100000,
'max_bin':255,
'verbose':-1,
'random_state': 47,
'early_stopping_rounds':100,
}<train_model>
total.drop(['Name','Ticket','Cabin'],axis=1,inplace=True)
 total=pd.get_dummies(total,columns=['SibSp','Parch','Age_cat','Title','FamilySize','Fare_cat','FamilySize_cat'])
 total['Age']=total['Age'].astype(int )
NFOLDS = 10
folds = KFold(n_splits=NFOLDS)
 splits = folds.split(X, y)
 y_preds = np.zeros(test.shape[0])
 y_oof = np.zeros(X.shape[0])
 score = 0
for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label = y_train)
 dvalid = lgb.Dataset(X_valid, label = y_valid)
 clf = lgb.train(params, dtrain, 10000, valid_sets = [dtrain, dvalid],
verbose_eval = 200, early_stopping_rounds = 500)
 y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clf.predict(test)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
print("Mean AUC: ", score)
 print("Out of folds AUC: ", roc_auc_score(y, y_oof))<save_to_csv>
train=total[:len(train)]
test=total[len(train):]
np.random.seed(42)
 X_train, X_test, y_train, y_test = train_test_split(train,target, test_size = 0.2)
 models = {"KNN": KNeighborsClassifier() ,
"Logistic Regression": LogisticRegression(max_iter=10000),
"Random Forest": RandomForestClassifier() ,
"SVC" : SVC(probability=True),
"DecisionTreeClassifier" : DecisionTreeClassifier() ,
"AdaBoostClassifier" : AdaBoostClassifier() ,
"GradientBoostingClassifier" : GradientBoostingClassifier() ,
"GaussianNB" : GaussianNB() ,
"LinearDiscriminantAnalysis" : LinearDiscriminantAnalysis() ,
"QuadraticDiscriminantAnalysis" : QuadraticDiscriminantAnalysis() }
def fit_and_score(models, X_train, X_test, y_train, y_test):
np.random.seed(42)
 model_scores = {}
for name, model in models.items() :
model.fit(X_train, y_train)
 y_pred = model.predict(X_test)
 model_scores[name] = roc_auc_score(y_pred, y_test)
 return model_scores
model_scores = fit_and_score(models=models,
X_train=X_train,
X_test=X_test,
y_train=y_train,
y_test=y_test)
 model_scores
<load_from_csv><EOS> 
leaks = {
897:1,
899:1,
930:1,
932:1,
949:1,
987:1,
995:1,
998:1,
999:1,
1016:1,
1047:1,
1083:1,
1097:1,
1099:1,
1103:1,
1115:1,
1118:1,
1135:1,
1143:1,
1152:1,
1153:1,
1171:1,
1182:1,
1192:1,
1203:1,
1233:1,
1250:1,
1264:1,
1286:1,
935:0,
957:0,
972:0,
988:0,
1004:0,
1006:0,
1011:0,
1105:0,
1130:0,
1138:0,
1173:0,
1284:0,
}
model=GradientBoostingClassifier()
model.fit(train,target)
 sub = pd.DataFrame()
sub['PassengerId'] = test['PassengerId']
sub['Survived'] = model.predict(test)
 sub['Survived'] = sub['Survived'].apply(lambda x: 1 if x>0.8 else 0)
 sub['Survived'] = sub.apply(lambda r: leaks[int(r['PassengerId'])] if int(r['PassengerId'])in leaks else r['Survived'], axis=1)
 sub.to_csv('sub_titan.csv', index=False )