Kaggle competition: Titanic - Machine Learning from Disaster
for col in {'pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude'}:
train_df['rough' + col] = train_df[col].round(2 )<groupby>
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv' )
a=train_df.groupby(['roughpickup_latitude','roughpickup_longitude'])[['pickup_latitude', 'pickup_longitude']].agg(['mean','count'] )<sort_values>
seed = 42
a.columns = ['mean_pickup_latitude','c1','mean_pickup_longitude','c2']
a.head()
a.sort_values(['c1','c2'],ascending=False ).head() 
<groupby>
train = train.drop('Cabin',axis = 1)
 train = train.drop('Name',axis = 1)
 train = train.drop('Ticket',axis = 1)
 train = train.drop('PassengerId',axis = 1)
 train = train.reset_index(drop=True )
b=train_df.groupby(['roughdropoff_latitude','roughdropoff_longitude'])[['dropoff_latitude', 'dropoff_longitude']].agg(['mean','count'] )<sort_values>
for col in train.columns:
if train[col].dtype == 'O':
train[col] = pd.get_dummies(train[col])
 train.head()
b.columns = ['mean_dropoff_latitude','c1','mean_dropoff_longitude','c2']
b.head()
b.sort_values(['c1','c2'],ascending=False ).head(n=20)
<sort_values>
train = train.fillna(-9999 )
b[ b['mean_dropoff_latitude']<40.7].sort_values(['c1','c2'],ascending=False ).head() 
<rename_columns>
X = train.iloc[:,1:]
y = train.iloc[:,0]
x_train, x_test,y_train, y_test = train_test_split(X,y,test_size = 0.2,random_state = seed )
a = a[['c1']].reset_index().rename(columns={'c1':'pickup_busyness'})
 b = b[['c1']].reset_index().rename(columns={'c1':'dropoff_busyness'} )<merge>
import xgboost as xgb
import optuna
train_df = pd.merge(train_df,a, how='left')
 train_df = pd.merge(train_df,b, how='left' )<prepare_x_and_y>
def objective(trial):
dtrain = xgb.DMatrix(x_train,label = y_train)
 dtest = xgb.DMatrix(x_test,label = y_test)
 num_round = 1000
param = {
"eta": trial.suggest_float('eta',1e-3, 0.3),
"objective": "binary:logistic",
"eva_metric":'auc',
"max_depth": trial.suggest_int('max_depth',4,16),
"subsample": trial.suggest_float('subsample',0.5,1),
"lambda":trial.suggest_float('lambda',1,3),
'feature_fraction':trial.suggest_float('feature_fraction',0.5,1),
"missing":-9999
}
bst = xgb.train(param, dtrain, evals=[(dtest, "validation")])
 preds = bst.predict(dtest)
 pred_labels = np.rint(preds)
 accuracy = sklearn.metrics.accuracy_score(y_test, pred_labels)
 return accuracy
X = train_df[['distance','year','month','day','hour','pickup_busyness','dropoff_busyness']].values
Y = train_df['fare_amount'].values<import_modules>
if __name__ == "__main__":
study = optuna.create_study(
 pruner=optuna.pruners.MedianPruner(n_warmup_steps=5), direction="maximize" 
)
 study.optimize(objective, n_trials=100)
 print(study.best_trial )
from sklearn.ensemble import RandomForestRegressor<import_modules>
params={'eta': 0.001832623843952462, 'max_depth': 4, 'subsample': 0.8827284715195801, 'lambda': 7.80011028366904}
from sklearn.ensemble import RandomForestRegressor<choose_model_class>
num_round = 100
dtrain = xgb.DMatrix(x_train,label = y_train)
 dtest = xgb.DMatrix(x_test,label = y_test)
 clf = xgb.XGBClassifier(**params)
 kfold = StratifiedKFold(n_splits = 5)
 results = cross_val_score(clf,x_train,y_train,cv = kfold)
 avg_results = results.sum() /5
kwargs = {'bootstrap': True,
'max_depth': None,
'max_features': 3,
'min_samples_leaf': 9,
'min_samples_split': 2}
rand_regr = RandomForestRegressor(n_estimators=20, **kwargs )<train_model>
num_rounds = 10
bst = xgb.train(params,dtrain,num_rounds )
rand_regr.fit(X, Y )<predict_on_test>
lgb_train = lgb.Dataset(x_train,y_train)
 lgb_test = lgb.Dataset(x_test,y_test )
y_pred = rand_regr.predict(X)
 print('chi squared rand forest with date %s' %(np.sum(( Y-y_pred)**2.) /len(Y)) **0.5 )<compute_test_metric>
def lightgbm_objective(trial):
lgb_train = lgb.Dataset(x_train,y_train)
 lgb_test = lgb.Dataset(x_test,y_test)
 param = {
'boost_type': trial.suggest_categorical('boost_type',['dart','gbdt']),
"eta": trial.suggest_float('eta',1e-3, 0.3),
"objective": "binary:logistic",
"eva_metric":'auc',
"max_depth": trial.suggest_int('max_depth',4,16),
"subsample": trial.suggest_float('subsample',0.5,1),
"lambda":trial.suggest_float('lambda',1,20),
'feature_fraction':trial.suggest_float('feature_fraction',0.5,1),
"num_boost_round":trial.suggest_int('num_boost_round',20,100)
 }
gbm = lgb.train(params,
lgb_train,
num_boost_round=100,
valid_sets=lgb_test, 
)
 preds = gbm.predict(x_test,num_iteration = gbm.best_iteration)
 accuracy = sklearn.metrics.accuracy_score([int(round(x)) for x in preds],y_test)
 return accuracy
rand_regr.score(X,Y )<load_from_csv>
if __name__ == "__main__":
study = optuna.create_study(
 pruner=optuna.pruners.MedianPruner(n_warmup_steps=5), direction="maximize" 
)
 study.optimize(lightgbm_objective, n_trials=80)
 print(study.best_trial )
test_df = pd.read_csv('.. /input/test.csv' )<compute_test_metric>
params={'boost_type': 'dart', 'eta': 0.21195106328946775, 'max_depth': 9, 'subsample': 0.6526587206109331, 'lambda': 11.46556912870986, 'feature_fraction': 0.8104196182900097, 'num_boost_round': 88}
test_df['distance'] = haversine_np(test_df['pickup_longitude'], test_df['pickup_latitude'],
test_df['dropoff_longitude'], test_df['dropoff_latitude'] )<data_type_conversions>
gbm = lgb.train(params,
lgb_train,
valid_sets=lgb_test, 
)
 preds = gbm.predict(x_test,num_iteration = gbm.best_iteration)
 accuracy = sklearn.metrics.accuracy_score([int(round(x)) for x in preds],y_test)
 accuracy
test_df['pickup_datetime'] = pd.to_datetime(test_df['pickup_datetime'] )<feature_engineering>
lgb.cv(params = params,train_set = lgb_train,metrics = 'auc',nfold = 3 )
test_df['year'] = test_df['pickup_datetime'].dt.year
test_df['month'] = test_df['pickup_datetime'].dt.month
test_df['day'] = test_df['pickup_datetime'].dt.day
test_df['hour'] = test_df['pickup_datetime'].dt.hour
test_df['minute'] = test_df['pickup_datetime'].dt.minute<feature_engineering>
y_train.to_numpy()
for col in {'pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude'}:
test_df['rough' + col] = test_df[col].round(2 )<merge>
nn_model = tf.keras.Sequential()
nn_model.add(Dense(units = 9, kernel_initializer = 'uniform', activation = 'relu'))
nn_model.add(Dense(units = 9, kernel_initializer = 'uniform', activation = 'relu'))
nn_model.add(Dense(units = 5, kernel_initializer = 'uniform', activation = 'relu'))
nn_model.add(Dense(units = 1, kernel_initializer = 'uniform', activation = 'sigmoid'))
nn_model.compile(optimizer = 'adam',loss = 'binary_crossentropy',metrics = ['accuracy'])
 nn_model.fit(x_train,y_train, epochs=100)
 test_loss, test_acc = nn_model.evaluate(x_test,y_test, verbose=2 )
test_df = pd.merge(test_df,a, how='left')
 test_df = pd.merge(test_df,b, how='left' )<data_type_conversions>
test = test.drop('Cabin',axis = 1)
 test = test.drop('Name',axis = 1)
 test = test.drop('Ticket',axis = 1)
 test = test.drop('PassengerId',axis = 1)
 for col in test.columns:
if test[col].dtype == 'O':
test[col] = pd.get_dummies(test[col])
 test = test.fillna(-9999 )
test_df['pickup_busyness'] = test_df['pickup_busyness'].fillna(1)
 test_df['dropoff_busyness'] = test_df['dropoff_busyness'].fillna(1 )<predict_on_test>
test_xgb = xgb.DMatrix(test)
 pred_xgb = bst.predict(test_xgb)
 pred_xgb = [round(x)for x in pred_xgb]
X_to_pred = test_df[['distance','year','month','day','hour','pickup_busyness','dropoff_busyness']].values
y_pred = rand_regr.predict(X_to_pred )<save_to_csv>
test_lgb = lgb.Dataset(test)
 pred_lgb = gbm.predict(test)
 pred_lgb = [round(x)for x in pred_lgb]
submission = pd.DataFrame(
 {'key': test_df.key, 'fare_amount': y_pred},
columns = ['key', 'fare_amount'])
 submission.to_csv('submission.csv', index = False )<prepare_x_and_y>
pred_nn = nn_model.predict(test)
 pred_nn = [int(round(x)) for x in pred_nn.flat[:]]
<set_options>
Results = [round(( a+b+c)/3)for(a,b,c)in zip(pred_nn,pred_lgb,pred_xgb)]
%matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 pd.set_option('display.max_columns', None)
 train.head(5)
 test.head(5)
 print("The train data size before dropping Id feature is : {} ".format(train.shape))
print("The test data size before dropping Id feature is : {} ".format(test.shape))
print("*" * 70)
 train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True)
 print("The train data size after dropping Id feature is : {} ".format(train.shape))
print("The test data size after dropping Id feature is : {} ".format(test.shape))
ax = sns.scatterplot(x = train['GrLivArea'], y = train['SalePrice'], hue = train['OverallCond'],)
 sns.jointplot(x='GrLivArea', y='SalePrice', data= train)
 train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index)
 ax = sns.scatterplot(x = train['GrLivArea'], y = train['SalePrice'], hue = train['OverallCond'],)
 sns.jointplot(x='GrLivArea', y='SalePrice', data= train)
 sns.distplot(train['SalePrice'] , fit=norm, color = 'b'); 
(mu, sigma)= norm.fit(train['SalePrice'])
 print('
mu = {:.2f} and sigma = {:.2f}
'.format(mu, sigma))
plt.legend(['Normal dist.( $\mu=$ {:.2f} and $\sigma=$ {:.2f})'.format(mu, sigma)],
loc='best')
 plt.ylabel('Frequency')
 plt.title('SalePrice distribution')
 fig = plt.figure()
res = stats.probplot(train['SalePrice'], plot=plt)
 plt.show()
train["SalePrice"] = np.log1p(train["SalePrice"])
 sns.distplot(train['SalePrice'] , fit=norm, color= 'b'); 
(mu, sigma)= norm.fit(train['SalePrice'])
 print('
mu = {:.2f} and sigma = {:.2f}
'.format(mu, sigma))
plt.legend(['Normal dist.( $\mu=$ {:.2f} and $\sigma=$ {:.2f})'.format(mu, sigma)],
loc='best')
 plt.ylabel('Frequency')
 plt.title('SalePrice distribution')
 fig = plt.figure()
res = stats.probplot(train['SalePrice'], plot=plt)
 plt.show()
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
house_data = pd.concat(( train, test)).reset_index(drop=True)
 house_data.drop(['SalePrice'], axis=1, inplace=True)
 print("house_data size is : {}".format(house_data.shape))
data_null =(house_data.isnull().sum() / len(house_data)) * 100
data_null = data_null.drop(data_null[data_null == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :data_null})
 missing_data.head(20)
 f, ax = plt.subplots(figsize=(20, 12))
plt.xticks(rotation='60')
 sns.barplot(x=data_null.index, y=data_null, color = 'b')
 plt.xlabel('Features', fontsize=15)
 plt.ylabel('Percent of missing values', fontsize=15)
 plt.title('Percent missing data by feature', fontsize=15)
 corrmat = train.corr()
plt.subplots(figsize=(35,35))
sns.heatmap(corrmat, vmax=0.9, square=True, annot=True)
 house_data["PoolQC"] = house_data["PoolQC"].fillna("None")
 house_data["MiscFeature"] = house_data["MiscFeature"].fillna("None")
 house_data["Alley"] = house_data["Alley"].fillna("None")
 house_data["Fence"] = house_data["Fence"].fillna("None")
 house_data["FireplaceQu"] = house_data["FireplaceQu"].fillna("None")
 house_data["LotFrontage"] = house_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
house_data[col] = house_data[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
house_data[col] = house_data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
house_data[col] = house_data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
house_data[col] = house_data[col].fillna('None')
 house_data["MasVnrType"] = house_data["MasVnrType"].fillna("None")
 house_data["MasVnrArea"] = house_data["MasVnrArea"].fillna(0)
 house_data['MSZoning'] = house_data['MSZoning'].fillna(house_data['MSZoning'].mode() [0])
 house_data = house_data.drop(['Utilities'], axis=1)
 house_data["Functional"] = house_data["Functional"].fillna("Typ")
 house_data['Electrical'] = house_data['Electrical'].fillna(house_data['Electrical'].mode() [0])
 house_data['KitchenQual'] = house_data['KitchenQual'].fillna(house_data['KitchenQual'].mode() [0])
 house_data['Exterior1st'] = house_data['Exterior1st'].fillna(house_data['Exterior1st'].mode() [0])
 house_data['Exterior2nd'] = house_data['Exterior2nd'].fillna(house_data['Exterior2nd'].mode() [0])
 house_data['SaleType'] = house_data['SaleType'].fillna(house_data['SaleType'].mode() [0])
 house_data['MSSubClass'] = house_data['MSSubClass'].fillna("None")
 data_null =(house_data.isnull().sum() / len(house_data)) * 100
data_null = data_null.drop(data_null[data_null == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :data_null})
 missing_data.head()
house_data['MSSubClass'] = house_data['MSSubClass'].apply(str)
 house_data['OverallCond'] = house_data['OverallCond'].astype(str)
 house_data['YrSold'] = house_data['YrSold'].astype(str)
 house_data['MoSold'] = house_data['MoSold'].astype(str)
 cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(house_data[c].values))
house_data[c] = lbl.transform(list(house_data[c].values))
print('Shape all_data: {}'.format(house_data.shape))
house_data['TotalSF'] = house_data['TotalBsmtSF'] + house_data['1stFlrSF'] + house_data['2ndFlrSF']
numeric_feats = house_data.dtypes[house_data.dtypes != "object"].index
skewed_feats = house_data[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
 print("
Skew in numerical features:
")
 skewness = pd.DataFrame({'Skew' :skewed_feats})
 skewness.head(10)
 skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
house_data[feat] = boxcox1p(house_data[feat], lam)
 house_data = pd.get_dummies(house_data)
 print(house_data.shape)
 train = house_data[:ntrain]
test = house_data[ntrain:]
n_folds = 8
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse)
 lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1)
 averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features)
 stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))

 print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False)
<install_modules>
sub = pd.read_csv('/kaggle/input/titanic/test.csv')
 sub['Survived'] = [int(x)for x in Results]
sub = sub[['PassengerId','Survived']]
<set_options><EOS> 
my_sub = sub.to_csv('my_sub.csv',index = False )