Kaggle competition: Titanic - Machine Learning from Disaster
train['totalDistance'] = train['rideDistance'] + train['walkDistance'] + train['swimDistance']
train['killsWithoutMoving'] =(( train['kills'] > 0)&(train['totalDistance'] == 0))<feature_engineering>
gs = pd.read_csv("/kaggle/input/titanic/gender_submission.csv" )
train['headshot_rate'] = train['headshotKills'] / train['kills']
train['headshot_rate'] = train['headshot_rate'].fillna(0 )<drop_column>
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 train.head()
train.drop(train[train['killsWithoutMoving'] == True].index, inplace=True )<filter>
test = pd.read_csv("/kaggle/input/titanic/test.csv")
 test.head()
train[train['roadKills'] > 10]<drop_column>
train.isnull().sum()
train.drop(train[train['roadKills'] > 10].index, inplace=True )<filter>
train.isnull().sum()
display(train[train['kills'] > 30].shape)
 train[train['kills'] > 30]<drop_column>
test.isnull().sum()
train.drop(train[train['kills'] > 30].index, inplace=True )<drop_column>
test.isnull().sum()
train.drop(train[train['longestKill'] >= 1000].index, inplace=True )<drop_column>
pd.pivot_table(train, index = 'Survived', values = ['Age','SibSp','Parch','Fare'] )
train.drop(train[train['walkDistance'] >= 10000].index, inplace=True )<filter>
pd.pivot_table(train, index = 'Survived', columns = 'Pclass', values = 'Ticket' ,aggfunc ='count' )
train[train['rideDistance'] >= 20000]<drop_column>
pd.pivot_table(train, index = 'Survived', columns = 'Sex', values = 'Ticket' ,aggfunc ='count' )
train.drop(train[train['rideDistance'] >= 20000].index, inplace=True )<filter>
pd.pivot_table(train, index = 'Survived', columns = 'Embarked', values = 'Ticket' ,aggfunc ='count' )
train[train['swimDistance'] >= 2000]<drop_column>
train['Fam'] = train['SibSp'] + train['Parch']
train.head()
train.drop(train[train['swimDistance'] >= 2000].index, inplace=True )<filter>
train[train['cabin_adv'] == 'T'].index
train.drop(index = 339, inplace = True)
 train.cabin_adv.value_counts()
train[train['weaponsAcquired'] >= 80]<drop_column>
train['numeric_ticket'] = train.Ticket.apply(lambda x: 1 if x.isnumeric() else 0)
 train['ticket_letters'] = train.Ticket.apply(lambda x: ''.join(x.split(' ')[:-1] ).replace('.','' ).replace('/','' ).lower() if len(x.split(' ')[:-1])>0 else 0)
 pd.set_option("max_rows", None)
 train['ticket_letters'].value_counts()
train.drop(train[train['weaponsAcquired'] >= 80].index, inplace=True )<drop_column>
train.Name.head(50)
 train['name_title'] = train.Name.apply(lambda x: x.split(',')[1].split('.')[0].strip())
 train['name_title'].value_counts()
train.drop(train[train['heals'] >= 40].index, inplace=True )<drop_column>
test['Fam'] = test['SibSp'] + test['Parch']
test['cabin_adv'] = test.Cabin.apply(lambda x: str(x)[0])
 test['numeric_ticket'] = test.Ticket.apply(lambda x: 1 if x.isnumeric() else 0)
 test['ticket_letters'] = test.Ticket.apply(lambda x: ''.join(x.split(' ')[:-1] ).replace('.','' ).replace('/','' ).lower() if len(x.split(' ')[:-1])>0 else 0)
 test.Name.head(50)
 test['name_title'] = test.Name.apply(lambda x: x.split(',')[1].split('.')[0].strip())
 test['cabin_adv'].value_counts()
train = train.drop(columns = ['matchType', 'Id', 'groupId', 'matchId'] )<prepare_x_and_y>
train.Age = train.Age.fillna(train.Age.median())
 train.Fare = train.Fare.fillna(train.Fare.median())
 train.dropna(subset=['Embarked'],inplace = True)
 train.drop(columns=['Name', 'PassengerId', 'Cabin'], inplace = True)
 train.head()
df = df_sample.drop(columns = ['winPlacePerc'])
 y = df_sample['winPlacePerc']<split>
test.Age = test.Age.fillna(train.Age.median())
 test.Fare = test.Fare.fillna(train.Fare.median())
 test.dropna(subset=['Embarked'],inplace = True)
 passid = test['PassengerId'].copy()
test.drop(columns=['Name', 'PassengerId', 'Cabin'], inplace = True)
 test.head()
def split_vals(a, n : int):
return a[:n].copy() , a[n:].copy()
val_perc = 0.12
n_valid = int(val_perc * sample)
 n_trn = len(df)-n_valid
raw_train, raw_valid = split_vals(df_sample, n_trn)
 X_train, X_valid = split_vals(df, n_trn)
 y_train, y_valid = split_vals(y, n_trn)
 print('Sample train shape: ', X_train.shape,
'Sample target shape: ', y_train.shape,
'Sample validation shape: ', X_valid.shape )<compute_train_metric>
df = pd.concat([train.assign(ind=1), test.assign(ind=0)])
 df_hot = pd.get_dummies(df[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Ticket', 'Fare', 'Embarked', 'Fam', 'cabin_adv', 'numeric_ticket', 'ticket_letters', 'name_title', 'ind']])
 df_hot.head()
def print_score(m : RandomForestRegressor):
res = ['mae train: ', mean_absolute_error(m.predict(X_train), y_train),
'mae val: ', mean_absolute_error(m.predict(X_valid), y_valid)]
if hasattr(m, 'oob_score_'): res.append(m.oob_score_)
 print(res )<train_model>
test_hot, train_hot = df_hot[df_hot["ind"].eq(0)], df_hot[df_hot["ind"].eq(1)]
m1 = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features='sqrt',
n_jobs=-1)
 m1.fit(X_train, y_train)
 print_score(m1 )<compute_train_metric>
scale = StandardScaler()
train_hot[['Age','SibSp','Parch','Fare', 'Fam']]= scale.fit_transform(train_hot[['Age','SibSp','Parch','Fare', 'Fam']])
 train_hot.head()
fi = rf_feat_importance(m1, df); fi[:10]<features_selection>
scale = StandardScaler()
test_hot[['Age','SibSp','Parch','Fare', 'Fam']]= scale.fit_transform(test_hot[['Age','SibSp','Parch','Fare', 'Fam']])
 test_hot.head()
to_keep = fi[fi.imp>0.005].cols
print('Significant features: ', len(to_keep))
to_keep<split>
Y_train = train['Survived']
Y_train.head()
df_keep = df[to_keep].copy()
X_train, X_valid = split_vals(df_keep, n_trn )<train_model>
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
m2 = RandomForestRegressor(n_estimators=80, min_samples_leaf=3, max_features='sqrt',
n_jobs=-1)
 m2.fit(X_train, y_train)
 print_score(m2 )<split>
xgb = XGBClassifier(random_state =1, learning_rate = 0.5, min_child_weight = 0.03)
 cv = cross_val_score(xgb,train_hot,Y_train,cv=5)
 print(cv)
 print(cv.mean() )
val_perc_full = 0.12
n_valid_full = int(val_perc_full * len(train))
n_trn_full = len(train)-n_valid_full
df_full = train.drop(columns = ['winPlacePerc'])
 y = train['winPlacePerc']
df_full = df_full[to_keep]
X_train, X_valid = split_vals(df_full, n_trn_full)
 y_train, y_valid = split_vals(y, n_trn_full)
 print('Sample train shape: ', X_train.shape,
'Sample target shape: ', y_train.shape,
'Sample validation shape: ', X_valid.shape )<train_model>
rfc = RandomForestClassifier(n_estimators=100, max_depth=3, random_state=2)
 cv = cross_val_score(rfc,train_hot,Y_train,cv=5)
 print(cv)
 print(cv.mean() )
m3 = RandomForestRegressor(n_estimators=60, min_samples_leaf=3, max_features=0.5, n_jobs=-1)
 m3.fit(X_train, y_train)
 print_score(m3 )<load_from_csv>
svc = SVC(probability = True)
 cv = cross_val_score(svc,train_hot,Y_train,cv=5)
 print(cv)
 print(cv.mean() )
test = pd.read_csv('.. /input/test_V2.csv' )<feature_engineering>
X_train, X_valid, y_train, y_valid = train_test_split(
 train_hot, Y_train, test_size=0.33, random_state=42 )
test['headshot_rate'] = test['headshotKills'] / test['kills']
test['headshot_rate'] = test['headshot_rate'].fillna(0)
 test['totalDistance'] = test['rideDistance'] + test['walkDistance'] + test['swimDistance']
test['playersJoined'] = test.groupby('matchId')['matchId'].transform('count')
 test['killsNorm'] = test['kills']*(( 100-test['playersJoined'])/100 + 1)
 test['damageDealtNorm'] = test['damageDealt']*(( 100-test['playersJoined'])/100 + 1)
 test['maxPlaceNorm'] = test['maxPlace']*(( 100-train['playersJoined'])/100 + 1)
 test['matchDurationNorm'] = test['matchDuration']*(( 100-test['playersJoined'])/100 + 1)
 test['healsandboosts'] = test['heals'] + test['boosts']
test['killsWithoutMoving'] =(( test['kills'] > 0)&(test['totalDistance'] == 0))
test_pred = test[to_keep].copy()
test_pred.fillna(0, inplace=True)
 test_pred.head()<save_to_csv>
svc = SVC(probability = True)
 svc.fit(X_train, y_train )
predictions = np.clip(a = m3.predict(test_pred), a_min = 0.0, a_max = 1.0)
 pred_df = pd.DataFrame({'Id' : test['Id'], 'winPlacePerc' : predictions})
 pred_df.to_csv("submission.csv", index=False )<define_variables>
xgb = XGBClassifier(random_state =1, learning_rate = 0.5, min_child_weight = 0.03)
 xgb.fit(X_train, y_train )
np.random.seed(7777 )<load_from_csv>
dl1 = Sequential()
dl1.add(Dense(1002, activation='relu'))
dl1.add(Dense(512, activation='relu'))
dl1.add(Dense(1, activation='sigmoid'))
dl1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 dl1.fit(X_train, y_train, epochs=150, batch_size=10)
 _, accuracy = dl1.evaluate(X_train, y_train)
 print('Accuracy: %.2f' %(accuracy*100))
train_df = pd.read_csv(".. /input/train_V2.csv")
 test_df = pd.read_csv(".. /input/test_V2.csv" )<categorify>
dl2 = Sequential()
dl2.add(Dense(1002, activation='relu'))
dl2.add(Dense(512, activation='relu'))
dl2.add(Dense(256, activation='relu'))
dl2.add(Dense(1, activation='sigmoid'))
dl2.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 dl2.fit(X_train, y_train, epochs=150, batch_size=10)
 _, accuracy = dl2.evaluate(X_train, y_train)
 print('Accuracy: %.2f' %(accuracy*100))
matchTyp = ['squad-fpp', 'duo', 'solo-fpp', 'squad', 'duo-fpp', 'solo',
'normal-squad-fpp', 'crashfpp', 'flaretpp', 'normal-solo-fpp',
'flarefpp', 'normal-duo-fpp', 'normal-duo', 'normal-squad',
'crashtpp', 'normal-solo']
mapping = {}
for i, j in enumerate(matchTyp):
mapping[i] = j
train_df["matchTypeMap"] = train_df["matchType"].apply(lambda x: ''.join(str(i)for i, j in mapping.items() if x == j)).map(np.int64)
 test_df["matchTypeMap"] = test_df["matchType"].apply(lambda x: ''.join(str(i)for i, j in mapping.items() if x == j)).map(np.int64 )<count_missing_values>
knn = KNeighborsClassifier(algorithm= 'auto', n_neighbors = 7,weights= 'uniform')
 knn.fit(X_train, y_train )
train_df.drop(["matchType"], axis =1, inplace=True)
 test_df.drop(["matchType"], axis =1, inplace=True)
 train_df.dropna(inplace = True)
 train_df.isnull().any().any()<prepare_x_and_y>
lr = LogisticRegression(max_iter = 2000,
solver = 'liblinear')
 lr.fit(X_train, y_train )
X = train_df.drop(["Id", "groupId", "matchId", "winPlacePerc"], axis = 1)
 y = train_df["winPlacePerc"]
test = test_df.drop(["Id", "groupId", "matchId"], axis = 1 )<split>
rf = RandomForestClassifier(random_state = 1)
 rf.fit(X_train, y_train )
X_train_s, X_val_s, y_train_s, y_val_s = train_test_split(X, y, test_size = 0.5)
 del X
del y<train_model>
valid_preds1 = svc.predict(X_valid)
 valid_preds2 = xgb.predict(X_valid)
 valid_preds_ur3 = dl1.predict(X_valid)
 valid_preds3 = [round(x[0])for x in valid_preds_ur3]
valid_preds_ur4 = dl2.predict(X_valid)
 valid_preds4 = [round(x[0])for x in valid_preds_ur4]
valid_preds5 = knn.predict(X_valid)
 valid_preds6 = lr.predict(X_valid)
 valid_preds7 = rf.predict(X_valid )
params = {
'learning_rate': 0.3,
'num_leaves': 20,
'feature_fraction': 0.9,
'min_data_in_leaf': 100,
'lambda_l2': 4,
'objective': 'regression_l2',
'metric': 'mae',
'seed': 123}
lgb_dataset = lightgbm.Dataset(X_train_s, y_train_s)
 lgb_valid = lightgbm.Dataset(X_val_s, y_val_s)
 lgb_model = lightgbm.train(params, lgb_dataset, num_boost_round=10000, valid_sets = lgb_valid, early_stopping_rounds=30, verbose_eval=100)
 pred1 = lgb_model.predict(X_val_s )<create_dataframe>
print("SVC:", accuracy_score(y_valid, valid_preds1))
print("XGB:", accuracy_score(y_valid, valid_preds2))
print("DL1:", accuracy_score(y_valid, valid_preds3))
print("DL2:", accuracy_score(y_valid, valid_preds4))
print("KNN:", accuracy_score(y_valid, valid_preds5))
print("LR:", accuracy_score(y_valid, valid_preds5))
print("RF:", accuracy_score(y_valid, valid_preds5))
kfold = 15
skf = KFold(n_splits=kfold, random_state=42)
 pred2 = pd.DataFrame()
pred2['winPlacePerc'] = np.zeros(len(X_val_s))<train_model>
test_preds1 = svc.predict(test_hot)
 test_preds2 = xgb.predict(test_hot)
 test_preds_ur3 = dl1.predict(test_hot)
 test_preds3 = [round(x[0])for x in test_preds_ur3]
test_preds_ur4 = dl2.predict(test_hot)
 test_preds4 = [round(x[0])for x in test_preds_ur4]
test_preds5 = knn.predict(test_hot)
 test_preds6 = lr.predict(test_hot)
 test_preds7 = rf.predict(test_hot )
cat_model = CatBoostRegressor(iterations=50, depth=3, learning_rate=0.1, loss_function='RMSE')
 cat_model.fit(X_train_s, y_train_s)
 pred3 = cat_model.predict(X_val_s )<predict_on_test>
stacked_predictions = np.column_stack([valid_preds1,valid_preds2,valid_preds3,valid_preds4, valid_preds5, valid_preds6, valid_preds7])
 stacked_test_predictions = np.column_stack([test_preds1,test_preds2,test_preds3,test_preds4, test_preds5, test_preds6, test_preds7] )
xgb_model = xgboost.XGBRegressor(max_depth=11)
 for i,(train_index, test_index)in enumerate(skf.split(X_train_s, y_train_s)) :
X_train, X_valid = X_train_s.iloc[train_index], X_train_s.iloc[test_index]
y_train, y_valid = y_train_s.iloc[train_index], y_train_s.iloc[test_index]
xgb_model.fit(X_train, y_train, eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='mae', early_stopping_rounds=100)
 vtest = xgb_model.predict(X_val_s)
 pred2['winPlacePerc'] += vtest/kfold 
<predict_on_test>
meta_model = LinearRegression()
meta_model.fit(stacked_predictions, y_valid)
 meta_predictions_ur = meta_model.predict(stacked_test_predictions)
 preds = [round(x)for x in meta_predictions_ur]
preds
test_pred_lgm = lgb_model.predict(test)
 test_pred_xgb = xgb_model.predict(test)
 test_pred_cat = cat_model.predict(test )<concatenate>
df = pd.DataFrame({'PassengerId': passid,
'Survived' : preds})
 df.Survived = df.Survived.astype(int)
 print(df.shape)
 df.head()
<choose_model_class><EOS> 
df.to_csv('submission_final.csv', index =False )