Kaggle competition: Titanic - Machine Learning from Disaster
print(len(train[train['Category'].isnull() ]))
train = train[train['Category'].isnull() == False]
print(len(train[train['Category'].isnull() ]))
print(len(test))<count_missing_values>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()
train.isnull().sum()<count_missing_values>
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.head()
train.isnull().sum()<drop_column>
missing_data=train_data.isnull().sum()
missing_data
if 'Descript' in train:
train = train.drop(['Descript'], axis=1)
 if 'Resolution' in train:
train = train.drop(['Resolution'], axis=1)
 train.head()<feature_engineering>
test_data.isnull().sum()
def rebuild_datetime(df):
df['Dates'] = pd.to_datetime(df['Dates'])
 df['Date'] = df['Dates'].dt.date
df['Hour'] = df['Dates'].dt.hour
df['Minute'] = df['Dates'].dt.minute
df['DayOfWeek'] = df['Dates'].dt.weekday
df['Month'] = df['Dates'].dt.month
df['Year'] = df['Dates'].dt.year
df['Block'] = df['Address'].str.contains('block', case=False)
 return df
train = rebuild_datetime(train)
 test = rebuild_datetime(test)
 print("wrong Dates(train):", len(train[train['Dates'].isnull() ]))
print("wrong Dates(test):", len(test[test['Dates'].isnull() ]))<feature_engineering>
train_data.drop(['Name','Ticket','Cabin'],axis=1,inplace=True)
 test_data.drop(['Name','Ticket','Cabin'],axis=1,inplace=True )
wrongxycnt = lambda df : len(df[(df['X'] == -120.5)&(df['Y'] == 90.0)])
 print(wrongxycnt(train))
print(wrongxycnt(test))
def fix_gps(df):
cnt = 0
d = df[(df['X'] == -120.5)&(df['Y'] == 90.0)]
for idx, row in d.iterrows() :
district = row['PdDistrict']
xys = df[df['PdDistrict'] == district][['X', 'Y']]
df.loc[idx, ['X']] = xys['X'].mean()
df.loc[idx, ['Y']] = xys['Y'].mean()
cnt = cnt + 1
print('cnt', cnt)
 def fix_gps_values() :
fix_gps(train)
 fix_gps(test)
 def drop_wrong_gps(df):
df = df.drop(df[df['X'] == -120.5].index)
 return df
fix_gps(train)
 fix_gps(test)
 print(wrongxycnt(train))
print(wrongxycnt(test))<drop_column>
train_data.isnull().sum()
train_data.head()
if 'Address' in train:
train = train.drop(['Address'], axis=1)
 if 'Address' in test:
test = test.drop(['Address'], axis=1 )<drop_column>
test_data.isnull().sum()
if 'Dates' in train:
train = train.drop(['Dates'], axis=1)
 if 'Date' in train:
train = train.drop(['Date'], axis=1)
 if 'Dates' in test:
test = test.drop(['Dates'], axis=1)
 if 'Date' in test:
test = test.drop(['Date'], axis=1)
<drop_column>
train_data['Age'] = train_data['Age'].fillna(train_data['Age'].mean())
 test_data['Age'] = test_data['Age'].fillna(test_data['Age'].mean())
 test_data['Fare'] = test_data['Fare'].fillna(test_data['Fare'].median() )
if 'DoWN' in train:
train = train.drop(['DoWN'], axis=1)
 if 'DoWN' in test:
test = test.drop(['DoWN'], axis=1)
 if 'datetime' in train:
train = train.drop(['datetime'], axis=1)
 if 'Year' in train:
train = train.drop(['Year'], axis=1)
 if 'datetime' in test:
test = test.drop(['datetime'], axis=1)
 if 'Year' in test:
test = test.drop(['Year'], axis=1)
 train.head()<data_type_conversions>
train_data.dropna(subset = ["Embarked"], inplace=True )
test_ids = test['Id'].astype('int' )<drop_column>
train_data = pd.get_dummies(train_data, columns=["Sex"], drop_first=True)
 train_data = pd.get_dummies(train_data, columns=["Embarked"],drop_first=True )
if "Id" in test:
test = test.drop(['Id'], axis=1 )<define_variables>
test_data = pd.get_dummies(test_data, columns=["Sex"], drop_first=True)
 test_data = pd.get_dummies(test_data, columns=["Embarked"],drop_first=True )
train_category = train['Category']<categorify>
y = train_data["Survived"]
X = train_data.drop(['Survived'], axis=1 )
if 'Category' in train:
train = train.drop(['Category'], axis=1)
 train_X = train
categoricals = ["PdDistrict"]
le_pdDistrict = LabelEncoder()
train_X['PdDistrict'] = le_pdDistrict.fit_transform(train_X['PdDistrict'])
 test['PdDistrict'] = le_pdDistrict.transform(test['PdDistrict'])
 le_category = LabelEncoder()
train_Y = le_category.fit_transform(train_category)
 num_category = len(list(le_category.classes_))<split>
sc=StandardScaler()
sc.fit(train_data.drop(['Survived', 'PassengerId'], axis = 1))
X_train = sc.transform(train_data.drop(['Survived', 'PassengerId'], axis = 1))
X_train
def show_feature_importance(df_X, df_Y):
params = {
'n_estimators' : 3,
'learning_rate' : 0.4,
'max_delta_step' : 0.9,
'min_data_in_leaf' : 21,
'max_bin' : 465,
'num_leaves' : 41,
}
_train_X, _val_X, _train_y, _val_y = train_test_split(df_X, df_Y)
 model = LGBMClassifier(objective='multiclass', num_class=num_category, n_estimators=200)
 model.set_params(**params)
 model.fit(_train_X, _train_y)
 perm = PermutationImportance(model ).fit(_val_X, _val_y)
 display(eli5.show_weights(perm, feature_names=_val_X.columns.tolist() , include_styles=False))
show_feature_importance(train_X, train_Y )<init_hyperparams>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.23, random_state = 5)
 model_LR = LogisticRegression(max_iter=5000)
 model_LR.fit(X_train, y_train)
 LR_predict = model_LR.predict(X_test)
 LR_score = model_LR.score(X_test,y_test)
 model_X = XGBClassifier(eta=0.1, n_estimators=50,
max_depth=5, subsample=0.6, colsample_bytree=0.7,objective= 'binary:logistic',
scale_pos_weight=1, seed=27)
 model_X.fit(X_train, y_train)
 X_predict = model_X.predict(X_test)
 X_score = model_X.score(X_test,y_test)
 rfc = RandomForestClassifier(n_estimators=6)
 rfc.fit(X_train, y_train)
 RFC_predict = rfc.predict(X_test)
 RFC_score = rfc.score(X_test,y_test)
 model_DTC = DecisionTreeClassifier(max_depth=7, min_samples_leaf=6, min_samples_split=2)
 model_DTC.fit(X_train, y_train)
 DTC_predict = model_DTC.predict(X_test)
 DTC_score = model_DTC.score(X_test,y_test)
 model_GB = GradientBoostingClassifier(random_state=10, n_estimators=1500,min_samples_split=100, max_depth=6)
 model_GB.fit(X, y)
 GB_predict = model_GB.predict(X_test)
 GB_score = model_GB.score(X_test,y_test)
 x1=LR_score, X_score, RFC_score, DTC_score, GB_score
print(x1)
 sns.distplot(a=x1, kde=True)
 plt.legend()
plt.title('Accuracy estimates of Logistic regression, XGB, decision trees, Random Forest, and Gradient Boosting' )
n_splits = 8
def get_param(learning_rate, max_delta_step, min_data_in_leaf, max_bin, num_leaves):
params = {'n_estimators' : 400,
'boosting_type' : 'gbdt',
'objective' : 'multiclass',
'max_delta_step': max_delta_step,
'min_data_in_leaf': int(min_data_in_leaf),
'max_bin': int(max_bin),
'num_leaves': int(num_leaves),
'learning_rate' : learning_rate,
'num_class' : num_category,
'early_stopping_rounds': 5,
}
return params
def opt_test_func(learning_rate, max_delta_step, min_data_in_leaf, max_bin, num_leaves):
params = get_param(learning_rate, max_delta_step, min_data_in_leaf, max_bin, num_leaves)
 print("params : ", params)
 acc, _ = train(params)
 return acc
def multiclass_logloss(predictions, labels, epsilon=1e-12):
predictions = np.clip(predictions, epsilon, 1.- epsilon)
 N = predictions.shape[0]
loss = -np.sum(labels*np.log(predictions+1e-9)) /N
return loss
def train_cv(params):
models = []
kfold = KFold(n_splits=n_splits, shuffle=True, random_state=7)
 all_predictions = np.zeros(( len(train_X), len(list(le_category.classes_))))
for train_idx, test_idx in kfold.split(train_X, train_Y):
X_train, y_train = train_X.loc[train_idx], train_Y[train_idx]
X_valid, y_valid = train_X.loc[test_idx], train_Y[test_idx]
train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=categoricals)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=categoricals)
 model = lgb.train(params, train_set=train_data, num_boost_round=120, valid_sets=[valid_data], verbose_eval=5)
 pred = model.predict(X_valid)
 all_predictions[test_idx] = pred
models.append(model)
 labels_one_hot = np.eye(len(list(le_category.classes_)))[train_Y]
loss = multiclass_logloss(all_predictions, labels_one_hot)
 print("validation multiclass logloss :", loss)
 return models, loss
def bo_eval_func(learning_rate, max_delta_step, min_data_in_leaf, max_bin, num_leaves):
params = get_param(learning_rate, max_delta_step, min_data_in_leaf, max_bin, num_leaves)
 _, loss = train_cv(params)
 return -loss
def get_optimized_hyperparameters() :

 bo_params = {
'learning_rate' :(0.01, 0.4),
'max_delta_step':(0.5, 2.5),
'min_data_in_leaf':(15, 45),
'max_bin':(200, 500),
'num_leaves':(20, 50),
}
optimizer = BayesianOptimization(bo_eval_func, bo_params, random_state=1030)
 with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 init_points = 16
n_iter = 16
optimizer.maximize(init_points = init_points, n_iter = n_iter, acq='ucb', xi=0.0, alpha=1e-6)
 return optimizer.max['params']
params = {
'learning_rate' : 0.4,
'max_delta_step' : 0.9,
'min_data_in_leaf' : 21,
'max_bin' : 465,
'num_leaves' : 41,
}
params = get_param(**params)
 models, loss = train_cv(params)
 print("train loss : ", loss )<save_to_csv>
gnb= GaussianNB()
gnb.fit(X_train, y_train)
 prediction = gnb.predict(X_test)
 cross_scores = cross_val_score(gnb,X_train,y_train,cv=8)
 print(cross_scores)
 sns.distplot(a=cross_scores, kde=True)
 plt.legend()
def predict(models, test):
preds = []
for model in models:
pred = model.predict(test)
 preds.append(pred)
 predsCnt = len(preds)
 preds = np.array(preds)
 preds = np.sum(preds, axis=0)/ predsCnt
return preds
pred = predict(models, test)
 submission = pd.DataFrame(pred, columns=le_category.inverse_transform(np.linspace(0, 38, 39, dtype='int')) , index=test.index)
 submission.to_csv('submission.csv', index_label='Id' )<feature_engineering>
neigh= KNeighborsClassifier(n_neighbors=5, leaf_size=30)
 neigh.fit(X_train, y_train)
 KN_predict = neigh.predict(X_test)
 cross_scores = cross_val_score(neigh,X_train,y_train,cv=8)
 print(cross_scores)
 print(accuracy_score(KN_predict, y_test))
warnings.filterwarnings("ignore")
 train = pd.read_csv('.. /input/train.csv', parse_dates=['Dates'])
 test = pd.read_csv('.. /input/test.csv', parse_dates=['Dates'], index_col='Id')
 train['Date'] = pd.to_datetime(train['Dates'].dt.date)
 train['n_days'] =(train['Date'] - train['Date'].min() ).apply(lambda x: x.days)
 train['Day'] = train['Dates'].dt.day
train['DayOfWeek'] = train['Dates'].dt.weekday
train['Month'] = train['Dates'].dt.month
train['Year'] = train['Dates'].dt.year
train['Hour'] = train['Dates'].dt.hour
train['Minute'] = train['Dates'].dt.minute
train['Block'] = train['Address'].str.contains('block', case=False ).apply(lambda x: 1 if x == True else 0)
 train['ST'] = train['Address'].str.contains('ST', case=False ).apply(lambda x: 1 if x == True else 0)
 train["X_Y"] = train["X"] - train["Y"]
train["XY"] = train["X"] + train["Y"]
test['Date'] = pd.to_datetime(test['Dates'].dt.date)
 test['n_days'] =(test['Date'] - test['Date'].min() ).apply(lambda x: x.days)
 test['Day'] = test['Dates'].dt.day
test['DayOfWeek'] = test['Dates'].dt.weekday
test['Month'] = test['Dates'].dt.month
test['Year'] = test['Dates'].dt.year
test['Hour'] = test['Dates'].dt.hour
test['Minute'] = test['Dates'].dt.minute
test['Block'] = test['Address'].str.contains('block', case=False ).apply(lambda x: 1 if x == True else 0)
 test['ST'] = test['Address'].str.contains('ST', case=False ).apply(lambda x: 1 if x == True else 0)
 test["X_Y"] = test["X"] - test["Y"]
test["XY"] = test["X"] + test["Y"]
le1 = LabelEncoder()
train['PdDistrict'] = le1.fit_transform(train['PdDistrict'])
 test['PdDistrict'] = le1.transform(test['PdDistrict'])
 le2 = LabelEncoder()
y= le2.fit_transform(train['Category'])
 le3 = LabelEncoder()
le3.fit(list(train['Address'])+ list(test['Address']))
train['Address'] = le3.transform(train['Address'])
 test['Address'] = le3.transform(test['Address'])
 train.drop(['Dates','Date','Descript','Resolution', 'Category'], 1, inplace=True)
 test.drop(['Dates','Date',], 1, inplace=True)
 model = LGBMClassifier(objective="multiclass", num_class=39, max_bin = 465, max_delta_step = 0.9,
learning_rate=0.4, num_leaves = 42, n_estimators=100,)
 model.fit(train, y, categorical_feature=["PdDistrict", "DayOfWeek"])
 preds = model.predict_proba(test)
 submission = pd.DataFrame(preds, columns=le2.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) , index=test.index)
 submission.to_csv('LGBM_final.csv', index_label='Id' )<import_modules>
predictions = model_X.predict(test_data)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )