Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<categorify>
print(os.listdir(".. /input"))
sns.set(style="ticks")
 pd.options.display.max_rows = 200
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 train_copy = train.copy()
print(train.info())
 print(train.isnull().sum() )
cat_features = [col for col in train.columns if train[col].dtype =='object']
cat_features = cat_features + ['MSSubClass','MoSold','YrSold']
num_features = [col for col in train.columns if train[col].dtype in ['int64', 'float64'] and col not in cat_features]
num_features.remove('Id')
 num_features.remove('SalePrice')
 target_enc = TargetEncoder(cols=cat_features)
 train_TE = train[num_features].join(target_enc.fit_transform(train[cat_features], train.SalePrice))
clf = ExtraTreesRegressor(random_state=0)
 h = clf.fit(train_TE, train.SalePrice)
 feature_imp = pd.DataFrame(sorted(zip(clf.feature_importances_,train_TE)) , columns=['Value','Feature'])
 plt.figure(figsize=(20,14))
sns.barplot(x='Value', y='Feature', data=feature_imp.sort_values(by='Value', ascending=False))
plt.title('Feature importance with ExtraTreesRegressor')
 del clf, h;<sort_values>
numeric_features = ['Age', 'Fare']
numeric_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='median')) , 
('scaler', StandardScaler())])
 categorical_features = ['Embarked', 'Sex', 'Pclass']
categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='constant',fill_value='Missing')) , 
('ohe', OneHotEncoder(handle_unknown='ignore')) ])
 preprocessor = ColumnTransformer(
 transformers=[ 
('num', numeric_transformer, numeric_features), 
('cat', categorical_transformer, categorical_features)])
 clf = Pipeline(steps=[('preprocessor', preprocessor), 
('classifier', LogisticRegression())])
 X=train.drop(columns=['Survived'])
 y=train['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
random_state=0)
 clf.fit(X_train, y_train)
 print("model score: %.3f" % clf.score(X_test, y_test))
cols_to_keep = list(feature_imp.sort_values(by='Value', ascending=False ).reset_index(drop=True ).loc[:59, 'Feature'])
 print('Keeping the first {:d} most informative features'.format(len(cols_to_keep)) )<train_model>
clf = Pipeline(steps=[('preprocessor', preprocessor), 
('classifier', RandomForestClassifier())])
 X=train.drop(columns=['Survived'])
 y=train['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
random_state=0)
 clf.fit(X_train, y_train)
 print("model score: %.3f" % clf.score(X_test, y_test))
outliers_id = set(train.loc[train.GrLivArea > 4500, 'Id'])
 print('Outliers saved in outliers_id' )<categorify>
afeatures = ['Age', 'Fare'] + list(clf['preprocessor'].transformers_[1][1]['ohe'].get_feature_names(categorical_features))
X_train = train.loc[~train.Id.isin(outliers_id), cols_to_keep].reset_index(drop=True)
 X_test = test[cols_to_keep]
cat_cols = [col for col in cols_to_keep if col in cat_features]
num_cols = [col for col in cols_to_keep if col not in cat_features]
lasso_preprocessor = ColumnTransformer(transformers=[ 
('num', PowerTransformer(standardize=True), num_cols), 
('cat', OneHotEncoder(handle_unknown='ignore'), cat_cols)
 ] )<categorify>
afeatures = ['Age', 'Fare','Embarked', 'Sex', 'Pclass']
oof_lasso = np.zeros(len(X_train))
preds_lasso = np.zeros(len(X_test))
out_preds_lasso = np.zeros(2)
 kf = KFold(n_splits=5, random_state=42, shuffle=True)
 print('Training {:d} Lasso models...
 '.format(kf.n_splits))
for i,(idxT, idxV)in enumerate(kf.split(X_train, y_train)) :
print('Fold', i)
 print(' rows of train =', len(idxT),'rows of holdout =', len(idxV))
train_prep = lasso_preprocessor.fit_transform(X_train.loc[idxT])
 valid_prep = lasso_preprocessor.transform(X_train.loc[idxV])
 out_prep = lasso_preprocessor.transform(train.loc[train.Id.isin(outliers_id), cols_to_keep])
 test_prep = lasso_preprocessor.transform(X_test)
 clf = Lasso(alpha=0.0005)
 h = clf.fit(train_prep, y_train.loc[idxT])
 oof_lasso[idxV] += clf.predict(valid_prep)
 out_preds_lasso += clf.predict(out_prep)/kf.n_splits
preds_lasso += clf.predict(test_prep)/kf.n_splits
del h, clf
print('
OOF CV:', mean_squared_error(y_train, oof_lasso, squared=False))<feature_engineering>
train['Cabin'].fillna('0' ).str[:1].value_counts()
lasso_submission = np.exp(preds_lasso)
 lasso_submission[1089] = train.loc[train.GrLivArea>4500, 'SalePrice'].mean()<categorify>
train['modCabin'] = train['Cabin'].fillna('0' ).str[:1]
test['modCabin'] = train['Cabin'].fillna('0' ).str[:1]
tree_preprocessor = ColumnTransformer(transformers=[ 
('num', 'passthrough', num_cols), 
('cat', TargetEncoder(cols=cat_cols), cat_cols)
 ] )<split>
train['Survived'].value_counts()
oof_LGBM = np.zeros(len(X_train))
preds_LGBM = np.zeros(len(X_test))
out_preds_LGBM = np.zeros(2)
 kf = KFold(n_splits=5, random_state=42, shuffle=True)
 print('Training {:d} LGBM models...
 '.format(kf.n_splits))
for i,(idxT, idxV)in enumerate(kf.split(X_train, y_train)) :
print('Fold', i)
 print(' rows of train =', len(idxT),'rows of holdout =', len(idxV))
train_prep = tree_preprocessor.fit_transform(X_train.loc[idxT], y_train.loc[idxT])
 valid_prep = tree_preprocessor.transform(X_train.loc[idxV])
 out_prep = tree_preprocessor.transform(train.loc[train.Id.isin(outliers_id), cols_to_keep])
 test_prep = tree_preprocessor.transform(X_test)
 clf = LGBMRegressor(n_estimators=5000, random_state=42, num_leaves=8, colsample_bytree=0.8)
 h = clf.fit(train_prep, y_train.loc[idxT],
eval_set=[(valid_prep, y_train.loc[idxV])],
verbose=300, early_stopping_rounds=200, eval_metric='rmse')
 oof_LGBM[idxV] += clf.predict(valid_prep)
 out_preds_LGBM += clf.predict(out_prep)/kf.n_splits
preds_LGBM += clf.predict(test_prep)/kf.n_splits
del h, clf
print('
OOF CV', mean_squared_error(y_train, oof_LGBM, squared=False))<feature_engineering>
train.groupby(['Survived'])['modCabin'].value_counts()
LGBM_submission = np.exp(preds_LGBM)
 LGBM_submission[1089] = train.loc[train.GrLivArea>4500, 'SalePrice'].mean()<save_to_csv>
train.groupby(['Survived'])['modCabin'].value_counts(normalize=True )
kaggle_sub = best_weight*lasso_submission +(1-best_weight)*LGBM_submission
output = pd.DataFrame({'Id': test.Id, 'SalePrice': kaggle_sub})
 output.to_csv('my_submission.csv', index=False)
 print('Your submission was successfully saved!' )<set_options>
vec = DictVectorizer(sparse=False, dtype=int)
 res = vec.fit_transform(train[['modCabin', 'Pclass', 'Age', 'SibSp', 'Parch','Embarked', 'Sex']].fillna('NA' ).to_dict(orient='records'))
res
%matplotlib inline
color = sns.color_palette()
sns.set_style("darkgrid")
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
pd.set_option(
 "display.float_format", lambda x: f"{x:.3f}" 
)
 print(
 check_output(["ls", ".. /input"] ).decode("utf8")
 )<load_from_csv>
df = pd.DataFrame(res, columns=vec.get_feature_names())
 df.head()
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<prepare_x_and_y>
corr_df = df.corr().abs().unstack().sort_values(ascending=False)
 corr_df[corr_df<1].drop_duplicates() [:10]
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(["SalePrice"], axis=1, inplace=True)
 print(f"all_data size is : {all_data.shape}" )<create_dataframe>
gbes = ensemble.GradientBoostingClassifier(n_estimators=400,
validation_fraction=0.2,
n_iter_no_change=5, tol=0.01,
random_state=0)
 X_train, X_test, y_train, y_test = train_test_split(train.Age, train.Survived, test_size=0.2,
random_state=0)
 gbes.fit(X_train.fillna(0 ).values.reshape(-1, 1),y_train)
 gbes.score(X_test.fillna(0 ).values.reshape(-1, 1),y_test )
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(
 ascending=False 
)[:30]
missing_data = pd.DataFrame({"Missing Ratio": all_data_na})
 missing_data.head(20 )<data_type_conversions>
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
all_data["PoolQC"] = all_data["PoolQC"].fillna("None" )<data_type_conversions>
X_train, X_test, y_train, y_test = train_test_split(df, train.Survived, test_size=0.2,
random_state=0)
 gbes.fit(X_train,y_train)
 gbes.score(X_test,y_test )
all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None" )<data_type_conversions>
print(cross_val_score(gbes, df, train.Survived, cv=3))
all_data["Alley"] = all_data["Alley"].fillna("None" )<data_type_conversions>
test_transf = vec.transform(test[['modCabin', 'Pclass', 'Age', 'SibSp', 'Parch','Embarked', 'Sex']].fillna('NA' ).to_dict(orient='records'))
df_test = pd.DataFrame(test_transf, columns=vec.get_feature_names())
 df_test.head()
all_data["Fence"] = all_data["Fence"].fillna("None" )<data_type_conversions>
gbespred = gbes.predict(df_test)
 out_df = pd.DataFrame({"PassengerId":test["PassengerId"].values})
 out_df['Survived'] = gbespred
out_df.to_csv("submission.csv", index=False)
 print('out_df.to_csv')
 print(out_df.head())
 out_df.Survived.value_counts()
all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None" )<categorify>
preprocessing.KBinsDiscretizer(n_bins=[2,2,2,2], encode='ordinal' ).fit_transform(train.loc[:,['Age','Fare','SibSp','Parch']].dropna() )
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median())
 )<data_type_conversions>
train.Embarked.value_counts()
for col in("GarageType", "GarageFinish", "GarageQual", "GarageCond"):
all_data[col] = all_data[col].fillna("None" )<data_type_conversions>
train.loc[train.Embarked.isnull() ,'Embarked']='Q'
for col in("GarageYrBlt", "GarageArea", "GarageCars"):
all_data[col] = all_data[col].fillna(0 )<data_type_conversions>
train['IsCabinNull'] = train.Cabin.isnull().astype(int)
 test['IsCabinNull'] = test.Cabin.isnull().astype(int )
for col in(
 "BsmtFinSF1",
"BsmtFinSF2",
"BsmtUnfSF",
"TotalBsmtSF",
"BsmtFullBath",
"BsmtHalfBath", 
):
all_data[col] = all_data[col].fillna(0 )<data_type_conversions>
train.loc[train.Age.isnull() , 'Age'] = 28
test.loc[test.Age.isnull() , 'Age'] = 28
for col in("BsmtQual", "BsmtCond", "BsmtExposure", "BsmtFinType1", "BsmtFinType2"):
all_data[col] = all_data[col].fillna("None" )<data_type_conversions>
train.loc[(train.Fare==0), 'Fare'] = train.Fare.median()
test.loc[(test.Fare==0), 'Fare'] = test.Fare.median()
test.loc[(test.Fare.isnull()), 'Fare'] = test.Fare.median()
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )<data_type_conversions>
train['age_log'] = np.log(train['Age'])
 test['age_log'] = np.log(test['Age'])
 train['fare_log'] = np.log(train['Fare'])
 test['fare_log'] = np.log(test['Fare'] )
all_data["MSZoning"] = all_data["MSZoning"].fillna(all_data["MSZoning"].mode() [0] )<drop_column>
scaler = StandardScaler()
scaler.fit(train[['Age']])
 train['age_scale'] = scaler.transform(train[['Age']])
 test['age_scale'] = scaler.transform(test[['Age']])
 scaler = StandardScaler()
scaler.fit(train[['Fare']])
 train['fare_scale'] = scaler.transform(train[['Fare']])
 test['fare_scale'] = scaler.transform(test[['Fare']] )
all_data = all_data.drop(["Utilities"], axis=1 )<data_type_conversions>
train_objs_num = len(train)
 dataset = pd.concat(objs=[train, test],sort=False)
 dataset['ngroup'] = dataset.groupby(['Pclass','Sex'] ).ngroup()
dataset = dataset['ngroup']
train['ngroup'] = copy.copy(dataset[:train_objs_num])
 test['ngroup'] = copy.copy(dataset[train_objs_num:] )
all_data["Functional"] = all_data["Functional"].fillna("Typ" )<data_type_conversions>
train['SibParchSum'] = train['SibSp'] + train['Parch']
test['SibParchSum'] = test['SibSp'] + test['Parch']
train['IsWithFamily'] =(train['SibParchSum']>0 ).astype(int)
 test['IsWithFamily'] =(test['SibParchSum']>0 ).astype(int)
 binner = KBinsDiscretizer(encode='ordinal')
 binner.fit(train[['Fare']])
 train['Fare_bins'] = binner.transform(train[['Fare']])
 test['Fare_bins'] = binner.transform(test[['Fare']])
 binner = KBinsDiscretizer(encode='ordinal')
 binner.fit(train[['Age']])
 train['age_bins'] = binner.transform(train[['Age']])
 test['age_bins'] = binner.transform(test[['Age']] )
all_data["Electrical"] = all_data["Electrical"].fillna(all_data["Electrical"].mode() [0] )<data_type_conversions>
encoder = OrdinalEncoder()
encoder.fit(train[['Sex']])
 train['sex_integer'] = encoder.transform(train[['Sex']])
 test['sex_integer'] = encoder.transform(test[['Sex']])
 encoder = OrdinalEncoder()
encoder.fit(train[['Embarked']])
 train['embarked_integer'] = encoder.transform(train[['Embarked']])
 test['embarked_integer'] = encoder.transform(test[['Embarked']] )
all_data["KitchenQual"] = all_data["KitchenQual"].fillna(all_data["KitchenQual"].mode() [0] )<categorify>
train_after_normalizer = preprocessing.Normalizer().fit_transform(train[['Age', 'Fare','SibSp', 'Parch','sex_integer','embarked_integer','Pclass']] )
all_data["Exterior1st"] = all_data["Exterior1st"].fillna(all_data["Exterior1st"].mode() [0])
 all_data["Exterior2nd"] = all_data["Exterior2nd"].fillna(all_data["Exterior2nd"].mode() [0] )<data_type_conversions>
train_after_normalizer = pd.DataFrame(train_after_normalizer, columns = ['Age', 'Fare','SibSp', 'Parch','sex_integer','embarked_integer','Pclass'] )
all_data["SaleType"] = all_data["SaleType"].fillna(all_data["SaleType"].mode() [0] )<data_type_conversions>
print(train.groupby('Pclass')['Fare'].mean())
 print(test.groupby('Pclass')['Fare'].mean())
 print(test[test.Fare.isnull() ])
all_data["MSSubClass"] = all_data["MSSubClass"].fillna("None" )<sort_values>
test.loc[test.Fare.isnull() ,'Fare'] = 13
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({"Missing Ratio": all_data_na})
 missing_data.head()<data_type_conversions>
test.loc[test.PassengerId==1044]
all_data["MSSubClass"] = all_data["MSSubClass"].apply(str)
 all_data["OverallCond"] = all_data["OverallCond"].astype(str)
 all_data["YrSold"] = all_data["YrSold"].astype(str)
 all_data["MoSold"] = all_data["MoSold"].astype(str )<define_variables>
test.groupby(['Pclass','Sex'])['Age'].mean()
cols =(
 "FireplaceQu",
"BsmtQual",
"BsmtCond",
"GarageQual",
"GarageCond",
"ExterQual",
"ExterCond",
"HeatingQC",
"PoolQC",
"KitchenQual",
"BsmtFinType1",
"BsmtFinType2",
"Functional",
"Fence",
"BsmtExposure",
"GarageFinish",
"LandSlope",
"LotShape",
"PavedDrive",
"Street",
"Alley",
"CentralAir",
"MSSubClass",
"OverallCond",
"YrSold",
"MoSold", 
)
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print(f"Shape all_data: {all_data.shape}" )<feature_engineering>
train['Survived'].value_counts(sort = False )
all_data["TotalSF"] = all_data["TotalBsmtSF"] + all_data["1stFlrSF"] + all_data["2ndFlrSF"]<feature_engineering>
total = train.isnull().sum().sort_values(ascending=False)
 percent =(train.isnull().sum() /train.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(5 )
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam)
<categorify>
total = test.isnull().sum().sort_values(ascending=False)
 percent =(test.isnull().sum() /test.isnull().count() ).sort_values(ascending=False)
 missing_test_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_test_data.head(5 )
all_data = pd.get_dummies(all_data)
 print(all_data.shape )<import_modules>
train.nunique()
train = all_data[:ntrain]
test = all_data[ntrain:] 
<import_modules>
y_train = train.Survived
from sklearn.linear_model import ElasticNet, Lasso
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import KFold, cross_val_score
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler<compute_train_metric>
train.groupby(train.Cabin.isnull() , as_index=False ).size()
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse = np.sqrt(
 -cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv=kf)
)
 return rmse<choose_model_class>
train['SibSp'].value_counts(sort=False )
lasso = make_pipeline(RobustScaler() , Lasso(alpha=0.0005, random_state=1))<choose_model_class>
train[['SibSp', 'Survived']].groupby('SibSp')['Survived'].sum()
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=0.9, random_state=3))<choose_model_class>
( train[['SibSp', 'Survived']].groupby('SibSp')['Survived'].sum() /train['SibSp'].value_counts(sort=False)) *100
KRR = KernelRidge(alpha=0.6, kernel="polynomial", degree=2, coef0=2.5 )<choose_model_class>
train.Parch.value_counts(sort=False )
GBoost = GradientBoostingRegressor(
 n_estimators=3000,
learning_rate=0.05,
max_depth=4,
max_features="sqrt",
min_samples_leaf=15,
min_samples_split=10,
loss="huber",
random_state=5,
)<choose_model_class>
train[['Parch', 'Survived']].groupby('Parch')['Survived'].sum()
model_xgb = xgb.XGBRegressor(
 colsample_bytree=0.4603,
gamma=0.0468,
learning_rate=0.05,
max_depth=3,
min_child_weight=1.7817,
n_estimators=2200,
reg_alpha=0.4640,
reg_lambda=0.8571,
subsample=0.5213,
silent=1,
random_state=7,
nthread=-1,
)<choose_model_class>
( train[['Parch', 'Survived']].groupby('Parch')['Survived'].sum() /train.Parch.value_counts(sort=False)) *100
model_lgb = lgb.LGBMRegressor(
 objective="regression",
num_leaves=5,
learning_rate=0.05,
n_estimators=720,
max_bin=55,
bagging_fraction=0.8,
bagging_freq=5,
feature_fraction=0.2319,
feature_fraction_seed=9,
bagging_seed=9,
min_data_in_leaf=6,
min_sum_hessian_in_leaf=11,
)<compute_test_metric>
( train[['Pclass', 'Survived']].groupby('Pclass')['Survived'].sum() /train.Pclass.value_counts(sort=False)) *100
score = rmsle_cv(lasso)
 print(f"
Lasso score: {score.mean() :.4f}({score.std() :.4f})
 " )<compute_test_metric>
train.Embarked.value_counts()
score = rmsle_cv(ENet)
 print(f"ElasticNet score: {score.mean() :.4f}({score.std() :.4f})
 " )<compute_test_metric>
const_cols = [c for c in train.columns if train[c].nunique(dropna=False)==1 ]
const_cols
score = rmsle_cv(KRR)
 print(f"Kernel Ridge score: {score.mean() :.4f}({score.std() :.4f})
 " )<compute_test_metric>
train.groupby('Pclass')['Age'].agg(['size', 'count', 'mean'] )
score = rmsle_cv(GBoost)
 print(f"Gradient Boosting score: {score.mean() :.4f}({score.std() :.4f})
 " )<compute_test_metric>
train.groupby('Sex')['Age'].agg(['size', 'count', 'mean'] )
score = rmsle_cv(model_xgb)
 print(f"Xgboost score: {score.mean() :.4f}({score.std() :.4f})
 " )<compute_test_metric>
train.groupby('Embarked')['Age'].agg(['size', 'count', 'mean'] )
score = rmsle_cv(model_lgb)
 print(f"LGBM score: {score.mean() :.4f}({score.std() :.4f})
 " )<train_model>
np.random.seed(2019)
 numeric_features = ['Age', 'Fare']
numeric_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='median')) , 
('scaler', StandardScaler())])
 categorical_features = ['Embarked', 'Sex', 'Pclass']
categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='constant', fill_value='missing')) , 
('onehot', OneHotEncoder(handle_unknown='ignore')) ])
 preprocessor = ColumnTransformer(
 transformers=[ 
('num', numeric_transformer, numeric_features), 
('cat', categorical_transformer, categorical_features)])
 clf = Pipeline(steps=[('preprocessor', preprocessor), 
('classifier', LogisticRegression(solver='lbfgs')) ])
 Xt_train, Xt_test, yt_train, yt_test = train_test_split(train, y_train,stratify=y_train, test_size=0.3)
 clf.fit(Xt_train, yt_train)
 print("model score: %.3f" % clf.score(Xt_test, yt_test))
print(cross_val_score(clf, train, y_train, cv=5))
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([model.predict(X)for model in self.models_])
 return np.mean(predictions, axis=1 )<compute_train_metric>
features= ['Age', 'Fare','SibSp', 'Parch','Sex','Embarked','Pclass']
num_features = ['SibSp','Parch']
num_features_to_scale = ['Age', 'Fare']
cat_features = ['Sex', 'Embarked','Pclass']
train_test_concat = pd.concat([train[features],test[features]] )
averaged_models = AveragingModels(models=(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(f" Averaged base models score: {score.mean() :.4f}({score.std() :.4f})
 " )<train_model>
scaler = StandardScaler()
scaler.fit(train_test_concat[num_features_to_scale] )
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack(
 [
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_
] 
)
 return self.meta_model_.predict(meta_features )<compute_train_metric>
train_scaled = train[num_features_to_scale].copy()
train_scaled = scaler.transform(train_scaled)
 train_scaled = pd.DataFrame(train_scaled,columns=num_features_to_scale)
 train_scaled = pd.concat([train_scaled,train[cat_features],train[num_features]],axis=1)
 print(train_scaled.info() )
stacked_averaged_models = StackingAveragedModels(
 base_models=(ENet, GBoost, KRR), meta_model=lasso 
)
 score = rmsle_cv(stacked_averaged_models)
 print(f"Stacking Averaged models score: {score.mean() :.4f}({score.std() :.4f})" )<compute_test_metric>
test_scaled = test[num_features_to_scale].copy()
test_scaled = scaler.transform(test_scaled)
 test_scaled = pd.DataFrame(test_scaled,columns=num_features_to_scale)
 test_scaled = pd.concat([test_scaled,test[cat_features],test[num_features]],axis=1)
 print(test_scaled.info() )
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<predict_on_test>
my_imputer = SimpleImputer(strategy='most_frequent')
 my_imputer.fit(pd.concat([train_scaled,test_scaled]))
data_with_imputed_values = pd.DataFrame(
 my_imputer.transform(train_scaled), columns=train_scaled.columns
).astype(train_scaled.dtypes.to_dict())
 test_data_with_imputed_values = pd.DataFrame(
 my_imputer.transform(test_scaled), columns=test_scaled.columns
).astype(test_scaled.dtypes.to_dict())
 print(data_with_imputed_values.info())
 print(test_data_with_imputed_values.info() )
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))<predict_on_test>
all_data = pd.concat([data_with_imputed_values,test_data_with_imputed_values])
 for column in all_data.select_dtypes(include=[np.object] ).columns:
data_with_imputed_values[column] = pd.Categorical(data_with_imputed_values[column], categories = all_data[column].unique())
 test_data_with_imputed_values[column] = pd.Categorical(test_data_with_imputed_values[column], categories = all_data[column].unique())
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))<predict_on_test>
data_with_imputed_values = pd.get_dummies(data_with_imputed_values,drop_first=True)
 test_data_with_imputed_values = pd.get_dummies(test_data_with_imputed_values,drop_first=True )
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))<compute_test_metric>
param_grid = {
'n_estimators' : [500,1200],
'max_depth': range(1,5,2),
'max_features' :('log2', 'sqrt'),
'class_weight':[{1: w} for w in [1,1.5]]
}
GridRF = GridSearchCV(RandomForestClassifier(random_state=15), param_grid)
 GridRF.fit(data_with_imputed_values, y_train)
 print(
 "
Best parameters
" + str(GridRF.best_params_))
rf = RandomForestClassifier(random_state=15,**GridRF.best_params_)
 rf.fit(data_with_imputed_values, y_train)
 Rfclf_fea = pd.DataFrame(rf.feature_importances_)
 Rfclf_fea["Feature"] = list(data_with_imputed_values.columns)
 Rfclf_fea.sort_values(by=0, ascending=False ).head(10 )
print("RMSLE score on train data:")
 print(rmsle(y_train, stacked_train_pred * 0.70 + xgb_train_pred * 0.15 + lgb_train_pred * 0.15))<define_variables>
cross_val_score(rf, data_with_imputed_values, y_train,cv=5 )
ensemble = stacked_pred * 0.70 + xgb_pred * 0.15 + lgb_pred * 0.15<save_to_csv>
one = OneHotEncoder(handle_unknown='ignore')
 simple = SimpleImputer(strategy='constant', fill_value='missing')
 pre = one.fit_transform(simple.fit_transform(train[['Embarked', 'Sex', 'Pclass']]))
sub = pd.DataFrame()
sub["Id"] = test_ID
sub["SalePrice"] = ensemble
sub.to_csv("submission.csv", index=False )<set_options>
def drop_cols_with_str(df,str_v):
return df[df.columns.drop(list(df.filter(regex=str_v)))]
data_with_imputed_values = drop_cols_with_str(data_with_imputed_values,'Cabin_')
 test_data_with_imputed_values = drop_cols_with_str(test_data_with_imputed_values,'Cabin_')
warnings.simplefilter("ignore")
 sns.set()
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))<load_from_csv>
X_t, X_v, y_t, y_v = train_test_split(data_with_imputed_values,y_train, stratify=y_train, test_size=0.2, random_state=2019)
 def runXGB(X_t, X_v, y_t, y_v, feature_names=None, seed_val=2017, num_rounds=200):
param = {}
param['objective'] = 'binary:logistic'
param['eta'] = 0.1
param['max_depth'] = 6
param['silent'] = 1
param['eval_metric'] = 'auc'
param['min_child_weight'] = 1
param['subsample'] = 0.5
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(X_t, label=y_t)
 xgtest = xgb.DMatrix(X_v, label=y_v)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=20)
 return model
model = runXGB(X_t, X_v, y_t, y_v)
train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 sample = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv" )<drop_column>
preds = model.predict(xgb.DMatrix(data_with_imputed_values))
train.drop("Id", axis=1, inplace=True)
 test.drop("Id", axis=1, inplace=True )<prepare_x_and_y>
pred_t = model.predict(xgb.DMatrix(data_with_imputed_values))
train_copy['Prediction'] =(pred_t > 0.4 ).astype(int)
 tn, fp, fn, tp = confusion_matrix(train_copy['Survived'], train_copy['Prediction'] ).ravel() 
(tn, fp, fn, tp )
target = train["SalePrice"]
train = train.drop("SalePrice", axis=1 )<count_missing_values>
train.isnull().sum().sum()<count_missing_values>
def runXGBcv(X, y, feature_names=None, seed_val=2017, num_rounds=100):
param = {}
param['objective'] = 'binary:logistic'
param['eta'] = 0.05
param['max_depth'] = 6
param['silent'] = 1
param['eval_metric'] = 'auc'
param['min_child_weight'] = 3
param['subsample'] = 0.5
param['colsample_bytree'] = 0.5
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(X, label=y)
 result = xgb.cv(plst, xgtrain, num_rounds, nfold=5,stratified=True,
folds=None, metrics=() , obj=None, feval=None, maximize=False,
early_stopping_rounds=20, fpreproc=None, as_pandas=True,
verbose_eval=None, show_stdv=True, seed=2019, callbacks=None, shuffle=True)
 return result
result = runXGBcv(data_with_imputed_values, y_train)
test.isnull().sum().sum()<categorify>
clf = xgb.XGBClassifier(
 max_depth = 7,
n_estimators=700,
learning_rate=0.1,
nthread=4,
subsample=1.0,
colsample_bytree=0.5,
min_child_weight = 3,
seed=1301)
 xgb_param = clf.get_xgb_params()
cvresult = xgb.cv(xgb_param, xgb.DMatrix(data_with_imputed_values,y_train),
num_boost_round=400, nfold=15, metrics=['auc'],
early_stopping_rounds=30,
stratified=True,
as_pandas = True, show_stdv= True, seed=1301)
 print('Best number of trees = {}'.format(cvresult.shape[0]))
clf.set_params(n_estimators=cvresult.shape[0])
 clf.fit(data_with_imputed_values, y_train, eval_metric='auc')
 print('Overall AUC:', roc_auc_score(y_train, clf.predict_proba(data_with_imputed_values)[:,1]))
test["MSZoning"].fillna("RL", inplace=True)
 train["LotFrontage"].fillna(0, inplace=True)
 test["LotFrontage"].fillna(0, inplace=True)
 train["Alley"].fillna("No_alley", inplace=True)
 test["Alley"].fillna("No_alley", inplace=True)
 train.drop("Utilities", axis=1, inplace=True)
 test.drop("Utilities", axis=1, inplace=True)
 test["Exterior1st"].fillna("VinylSd", inplace=True)
 test["Exterior2nd"].fillna("VinylSd", inplace=True)
 train["MasVnrType"].fillna("None", inplace=True)
 test["MasVnrType"].fillna("None", inplace=True)
 train["MasVnrArea"].fillna(0, inplace=True)
 test["MasVnrArea"].fillna(0, inplace=True)
 train["BsmtQual"].fillna("None", inplace=True)
 test["BsmtQual"].fillna("None", inplace=True)
 train["BsmtCond"].fillna("None", inplace=True)
 test["BsmtCond"].fillna("None", inplace=True)
 train["BsmtExposure"].fillna("None", inplace=True)
 test["BsmtExposure"].fillna("None", inplace=True)
 train["BsmtFinType1"].fillna("None", inplace=True)
 test["BsmtFinType1"].fillna("None", inplace=True)
 test["BsmtFinSF1"].fillna(0, inplace=True)
 train["BsmtFinType2"].fillna("None", inplace=True)
 test["BsmtFinType2"].fillna("None", inplace=True)
 test["BsmtFinSF2"].fillna(0, inplace=True)
 test["BsmtUnfSF"].fillna(0, inplace=True)
 test["TotalBsmtSF"].fillna(0, inplace=True)
 train["Electrical"].fillna("SBrkr", inplace=True)
 test["BsmtFullBath"].fillna(0, inplace=True)
 test["BsmtHalfBath"].fillna(0, inplace=True)
 test["KitchenQual"].fillna("TA", inplace=True)
 test["Functional"].fillna("Typ", inplace=True)
 train["FireplaceQu"].fillna("None", inplace=True)
 test["FireplaceQu"].fillna("None", inplace=True)
 train["GarageType"].fillna("None", inplace=True)
 test["GarageType"].fillna("None", inplace=True)
 train["GarageYrBlt"].fillna(0, inplace=True)
 test["GarageYrBlt"].fillna(0, inplace=True)
 train["GarageFinish"].fillna("None", inplace=True)
 test["GarageFinish"].fillna("None", inplace=True)
 test["GarageCars"].fillna(0, inplace=True)
 test["GarageArea"].fillna(0, inplace=True)
 train["GarageQual"].fillna("None", inplace=True)
 test["GarageQual"].fillna("None", inplace=True)
 train["GarageCond"].fillna("None", inplace=True)
 test["GarageCond"].fillna("None", inplace=True)
 train.drop("PoolQC", axis=1, inplace=True)
 test.drop("PoolQC", axis=1, inplace=True)
 train.drop("Fence", axis=1, inplace=True)
 test.drop("Fence", axis=1, inplace=True)
 train.drop("MiscFeature", axis=1, inplace=True)
 test.drop("MiscFeature", axis=1, inplace=True)
 test["SaleType"].fillna("WD", inplace=True )<categorify>
bclf = BalancedRandomForestClassifier(n_estimators=50, random_state=5)
 X_t, X_v, y_t, y_v = train_test_split(data_with_imputed_values,y_train, stratify=y_train, test_size=0.1, random_state=0)
 bclf.fit(X_t, y_t)
 print(classification_report(y_v, bclf.predict(X_v)))
<drop_column>
bclf.fit(data_with_imputed_values, y_train)
 print('Predict based on features in the test set')
 pred_bclf = bclf.predict(test_data_with_imputed_values)
train.drop("Condition2", axis=1, inplace=True)
 test.drop("Condition2", axis=1, inplace=True)
 train.drop("RoofMatl", axis=1, inplace=True)
 test.drop("RoofMatl", axis=1, inplace=True)
 train.drop("LowQualFinSF", axis=1, inplace=True)
 test.drop("LowQualFinSF", axis=1, inplace=True)
 def remode_flg(x):
if x["YearBuilt"]==x["YearRemodAdd"]:
res = 1
else:
res = 0
return res
train["Remod_flg"] = train.apply(remode_flg, axis=1)
 test["Remod_flg"] = test.apply(remode_flg, axis=1)
 train["YearRemodAdd"] = train["YrSold"] - train["YearRemodAdd"]
test["YearRemodAdd"] = test["YrSold"] - test["YearRemodAdd"]
train["GarageYrBlt"] = train["YrSold"] - train["GarageYrBlt"]
test["GarageYrBlt"] = test["YrSold"] - test["GarageYrBlt"]
train.drop(["YearBuilt", "YrSold"], axis=1, inplace=True)
 test.drop(["YearBuilt", "YrSold"], axis=1, inplace=True)
 train.drop("TotalBsmtSF", axis=1, inplace=True)
 test.drop("TotalBsmtSF", axis=1, inplace=True)
 train["FullBathrooms"] = train["BsmtFullBath"] + train["FullBath"]
train.drop(["BsmtFullBath", "FullBath"], axis=1, inplace=True)
 test["FullBathrooms"] = test["BsmtFullBath"] + test["FullBath"]
test.drop(["BsmtFullBath", "FullBath"], axis=1, inplace=True)
 train["HalfBathrooms"] = train["BsmtHalfBath"] + train["HalfBath"]
train.drop(["BsmtHalfBath", "HalfBath"], axis=1, inplace=True)
 test["HalfBathrooms"] = test["BsmtHalfBath"] + test["HalfBath"]
test.drop(["BsmtHalfBath", "HalfBath"], axis=1, inplace=True)
 train["Rooms"] = train["TotRmsAbvGrd"] - train["BedroomAbvGr"]
train.drop("TotRmsAbvGrd", axis=1, inplace=True)
 test["Rooms"] = test["TotRmsAbvGrd"] - test["BedroomAbvGr"]
test.drop("TotRmsAbvGrd", axis=1, inplace=True)
 train["Porch"] = train['WoodDeckSF'] + train['OpenPorchSF'] + train["EnclosedPorch"] + train["3SsnPorch"] + train["ScreenPorch"]
test["Porch"] = test['WoodDeckSF'] + test['OpenPorchSF'] + test["EnclosedPorch"] + test["3SsnPorch"] + test["ScreenPorch"]
train.drop(["WoodDeckSF", "OpenPorchSF", "EnclosedPorch", "3SsnPorch", "ScreenPorch"], axis=1, inplace=True)
 test.drop(["WoodDeckSF", "OpenPorchSF", "EnclosedPorch", "3SsnPorch", "ScreenPorch"], axis=1, inplace=True)
 train["PoolArea"] = train["PoolArea"].apply(lambda x:1 if x>0 else 0)
 test["PoolArea"] = test["PoolArea"].apply(lambda x:1 if x>0 else 0)
 train['Fireplaces'] = train['Fireplaces'].apply(lambda x:1 if x>0 else 0)
 test['Fireplaces'] = test['Fireplaces'].apply(lambda x:1 if x>0 else 0)
 log_col = ["LotFrontage", "LotArea", "MasVnrArea", "BsmtFinSF1", "BsmtFinSF2", "BsmtUnfSF",
"1stFlrSF", "2ndFlrSF", "GrLivArea", "GarageArea", "Porch"]
for c in log_col:
train[c] = np.log(train[c]+1)
 test[c] = np.log(test[c]+1 )<categorify>
X_t, X_v, y_t, y_v = train_test_split(data_with_imputed_values,y_train, stratify=y_train, test_size=0.1, random_state=0)
 dtrain = xgb.DMatrix(X_t, label=y_t)
 dtest = xgb.DMatrix(X_v, label=y_v)
 def xgb_evaluate(max_depth, gamma, colsample_bytree):
params = {'eval_metric': 'auc',
'max_depth': int(max_depth),
'subsample': 0.8,
'eta': 0.1,
'gamma': gamma,
'colsample_bytree': colsample_bytree}
cv_result = xgb.cv(params, dtrain, num_boost_round=100, nfold=3)
 return -1.0 * cv_result['test-auc-mean'].iloc[-1]
xgb_bo = BayesianOptimization(xgb_evaluate, {'max_depth':(3, 7),
'gamma':(0, 1),
'colsample_bytree':(0.3, 0.9)})
 xgb_bo.maximize(init_points=3, n_iter=5, acq='ei')
col_df = pd.DataFrame({"col":train.dtypes.index, "dtype":train.dtypes.values})
 obj_col = col_df[col_df["dtype"]=="object"]["col"].values
for i in obj_col:
ce_oe = ce.OneHotEncoder(cols=i, handle_unknown="impute")
 train = ce_oe.fit_transform(train)
 test = ce_oe.transform(test )<prepare_x_and_y>
params_xgb = xgb_bo.max['params']
params_xgb['max_depth'] = int(params_xgb['max_depth'])
 params_xgb
X_train = train
X_test = test
y_train = target<feature_engineering>
model2 = xgb.train(params_xgb, dtrain, num_boost_round=250)
 y_test_pred = model2.predict(dtest)>0.5
y_train_pred = model2.predict(dtrain)>0.5
print(classification_report(y_v, y_test_pred))
print(classification_report(y_t, y_train_pred))
model3 = xgb.train(params_xgb, xgb.DMatrix(data_with_imputed_values, label=y_train), num_boost_round=250)
 xgbpred =(model3.predict(xgb.DMatrix(test_data_with_imputed_values)) > 0.5 ).astype(int)
 out_df = pd.DataFrame({"PassengerId":test["PassengerId"].values})
 out_df['Survived'] = xgbpred
print(out_df.head())
 out_df.Survived.value_counts()
y_train = np.log10(y_train )<train_on_grid>
cb_model = CatBoostClassifier(
)
 cb_model.fit(X_t, y_t,
eval_set=(X_v,y_v),
use_best_model=True, 
)
 print(classification_report(y_v, cb_model.predict(X_v)))
 print(cb_model.get_params())
 cat_params = cb_model.get_params()
cb_model = CatBoostClassifier(**cat_params)
 cb_model.fit(data_with_imputed_values, y_train, 
)
 catpred = cb_model.predict(test_data_with_imputed_values)
%%time
tree = DecisionTreeRegressor(random_state=10)
 opt_tree = BayesSearchCV(
 tree,
{"max_depth":(5,20),
"max_leaf_nodes":(30,100)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_tree = opt_tree.fit(X_train, y_train)
 print("Best score:{}".format(bs_tree.best_score_))
print("Best params:{}".format(bs_tree.best_params_))<compute_train_metric>
clf.get_booster().get_score(importance_type='weight' )
%%time
tree = DecisionTreeRegressor(max_depth=9, max_leaf_nodes=67, random_state=10)
 print("Cross validation score")
 cr_score = cross_val_score(tree, X_train, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<train_on_grid>
pred_t = clf.predict(data_with_imputed_values, ntree_limit=cvresult.shape[0])
 train_copy['Prediction'] = pred_t
tn, fp, fn, tp = confusion_matrix(train_copy['Survived'], train_copy['Prediction'] ).ravel() 
(tn, fp, fn, tp )
%%time
forest = RandomForestRegressor(random_state=10)
 opt_forest = BayesSearchCV(
 forest,
{"max_depth":(5,40),
"max_leaf_nodes":(30,100),
"n_estimators":(50,100),
"min_samples_split":(0.00001,1.0),
"min_samples_leaf":(1,10)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_forest = opt_forest.fit(X_train, y_train)
 print("Best score:{}".format(bs_forest.best_score_))
print("Best params:{}".format(bs_forest.best_params_))<compute_train_metric>
preds_proba = clf.predict_proba(data_with_imputed_values, ntree_limit=cvresult.shape[0])
 train_copy['Prediction'] = preds_proba[:,1]
%%time
forest = RandomForestRegressor(n_estimators=100, max_depth=40, max_leaf_nodes=100, min_samples_leaf=1, min_samples_split=0.00001, random_state=10)
 print("Cross validation score")
 cr_score = cross_val_score(forest, X_train, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<train_model>
train_copy.Pclass.value_counts()