Kaggle competition: House Prices - Advanced Regression Techniques
df[df.Embarked.isnull() ]<data_type_conversions>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
df.Embarked = df.Embarked.fillna('C' )<count_values>
dataset=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv' )
df.Cabin = df.Cabin.fillna('0')
 len(df.Cabin.value_counts().index )<feature_engineering>
test=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
df['CabinCat'] = pd.Categorical(df.Cabin.apply(lambda x : x[0])).codes<import_modules>
dataset['GarageYrBlt'].isna().sum()
import re
from sklearn.preprocessing import LabelEncoder<feature_engineering>
dataset['TotRmsAbvGrd'].isna().sum()
df['Title'] = df.Name.apply(lambda x : re.search('([a-zA-Z]+)\.', x ).group(1))
title_mapping = {'Mr' : 1, 'Miss' : 2, 'Mrs' : 3, 'Master' : 4, 'Dr' : 5, 'Rev' : 6, 'Major' : 7, 'Col' : 7, 'Mlle' : 2, 'Mme' : 3, 'Don' : 9, 'Dona' : 9, 'Lady' : 10, 'Countess' : 10, 'Jonkheer' : 10, 'Sir' : 9, 'Capt' : 7, 'Ms' : 2}
df['TitleCat'] = df.Title.map(title_mapping)
 df['FamilySize'] = df.SibSp + df.Parch + 1
df['FamilyName'] = df.Name.apply(lambda x : str.split(x, ',')[0])
 df['IsAlone'] = 0
df.loc[df.FamilySize == 1, 'IsAlone'] = 1
le = LabelEncoder()
df['NameLength'] = df.Name.apply(lambda x : len(x))
df['NameLengthBin'] = pd.qcut(df.NameLength, 5)
 df['NameLengthBinCode'] = le.fit_transform(df.NameLengthBin)
 df['Embarked'] = pd.Categorical(df.Embarked ).codes
df = pd.concat([df, pd.get_dummies(df.Sex)], axis = 1)
 table_ticket = pd.DataFrame(df.Ticket.value_counts())
 table_ticket.rename(columns = {'Ticket' : 'TicketNum'}, inplace = True)
 table_ticket['TicketId'] = pd.Categorical(table_ticket.index ).codes
table_ticket.loc[table_ticket.TicketNum < 3, 'TicketId'] = -1
df = pd.merge(left = df, right = table_ticket, left_on = 'Ticket', right_index = True, how = 'left', sort = False)
 df['TicketCode'] = list(pd.cut(df.TicketId, bins = [-2, 0, 500, 1000], labels = [0, 1, 2]))
regex = re.compile('\s*(\w+)\s*')
 df['CabinNum'] = df.Cabin.apply(lambda x : len(regex.findall(x)) )<import_modules>
dataset['GarageCars'].isna().sum()
from sklearn.ensemble import ExtraTreesRegressor<find_best_model_class>
del dataset['GarageYrBlt']
del test['GarageYrBlt']
classers = ['Fare','Parch','Pclass','SibSp','TitleCat', 'CabinCat','female','male', 'Embarked', 'FamilySize', 'IsAlone', 'NameLengthBinCode','TicketNum','TicketCode']
etr = ExtraTreesRegressor(n_estimators = 200, random_state = 0)
 age_X_train = df[classers][df.Age.notnull() ]
age_y_train = df.Age[df.Age.notnull() ]
age_X_test = df[classers][df.Age.isnull() ]
etr.fit(age_X_train, np.ravel(age_y_train))
age_pred = etr.predict(age_X_test)
 df.loc[df.Age.isnull() , 'Age'] = age_pred<categorify>
import scipy.stats
childAge = 18
def getIdentity(passenger):
age, sex = passenger
if age < childAge:
return 'child'
elif sex == 'male':
return 'male_adult'
else:
return 'female_adult'
df = pd.concat([df, pd.DataFrame(df[['Age', 'Sex']].apply(getIdentity, axis = 1), columns = ['Identity'])], axis = 1)
 df = pd.concat([df, pd.get_dummies(df.Identity)], axis = 1 )<feature_engineering>
class column_similarity(object):
def __init__(self, data , filter_col):
num_columns=[]
cat_columns=[]
self.data=data.fillna('')
 for col in data.columns:
if self._check_numberic(data[col]):
num_columns.append(col)
 else:
cat_columns.append(col)
 self.cat_columns=np.array(cat_columns)
 self.num_columns=np.array(num_columns)
 self.cat_dist=None
self.num_dist=None
self.dist_reset()
def dist_reset(self):
if len(self.cat_columns)>0:
self.cat_dist=np.zeros(( len(self.cat_columns),len(self.cat_columns)))
 if len(self.num_columns)>0:
self.num_dist=np.zeros(( len(self.num_columns),len(self.num_columns)))
 def _check_numberic(self,data):
return pd.to_numeric(data[data.notna() ], errors='coerce' ).notnull().all()
def filtered_similarity(self,delimiter,threshold_chi,threshold_cor, filt=None):
self.dist_reset()
Cat_dist_frame=pd.DataFrame()
Num_dist_frame=pd.DataFrame()
chi_column1=[]
chi_column2=[]
chi_usecase1=[]
chi_usecase2=[]
chi_score=[]
if self.cat_columns is not None:
for i in range(len(self.cat_columns)) :
for j in range(i,len(self.cat_columns)) :
contingency_table=None
contingency_table=pd.crosstab(self.data[self.cat_columns[i]],self.data[self.cat_columns[j]])
 chi2, p, dof, ex=scipy.stats.chi2_contingency(contingency_table)
 self.cat_dist[i][j]=chi2
self.cat_dist[j][i]=chi2
if self.cat_dist[i][i]>0:
self.cat_dist[i]= self.cat_dist[i]/self.cat_dist[i][i]
selected_elems=list(np.where(self.cat_dist[i]>=threshold_chi)[0])
 if len(selected_elems)>0:
selected_elems.remove(i)
 chi_column1=chi_column1+[self.cat_columns[i]]*len(selected_elems)
 chi_column2=chi_column2+list(self.cat_columns[selected_elems])
 chi_score=chi_score+list(self.cat_dist[i][selected_elems])
 Cat_dist_frame['Column1']=[col.split(delimiter)[1] if delimiter in col else col for col in chi_column1]
Cat_dist_frame['Column2']=[col.split(delimiter)[1] if delimiter in col else col for col in chi_column2]
Cat_dist_frame['Score']=chi_score
return Cat_dist_frame.copy()
def similarity(self,delimiter='_',threshold_chi=0.0,threshold_cor=0.0):
global_Cat_dist_frame=self.filtered_similarity(delimiter,threshold_chi,threshold_cor)
 return global_Cat_dist_frame
DEFAULT_SURVIVAL_VALUE = 0.5
df['FamilySurvival'] = DEFAULT_SURVIVAL_VALUE
for _, grp_df in df.groupby(['FamilyName', 'Fare']):
if len(grp_df)!= 1 :
for index, row in grp_df.iterrows() :
smax = grp_df.drop(index ).Survived.max()
smin = grp_df.drop(index ).Survived.min()
pid = row.PassengerId
if smax == 1:
df.loc[df.PassengerId == pid, 'FamilySurvival'] = 1.0
elif smin == 0:
df.loc[df.PassengerId == pid, 'FamilySurvival'] = 0.0
for _, grp_df in df.groupby(['Ticket']):
if len(grp_df != 1):
for index, row in grp_df.iterrows() :
if(row.FamilySurvival == 0.0 or row.FamilySurvival == 0.5):
smax = grp_df.drop(index ).Survived.max()
smin = grp_df.drop(index ).Survived.min()
pid = row.PassengerId
if smax == 1:
df.loc[df.PassengerId == pid, 'FamilySurvival'] = 1.0
elif smin == 0:
df.loc[df.PassengerId == pid, 'FamilySurvival'] = 0.0
df.FamilySurvival.value_counts()<categorify>
sm=column_similarity(dataset,dataset.columns )
df['FareBin'] = pd.qcut(df.Fare, 5)
 le = LabelEncoder()
df['FareBinCode'] = le.fit_transform(df.FareBin )<categorify>
similarity=sm.similarity()
df['AgeBin'] = pd.qcut(df.Age, 4)
 le = LabelEncoder()
df['AgeBinCode'] = le.fit_transform(df.AgeBin )<import_modules>
from sklearn.preprocessing import MinMaxScaler, StandardScaler<define_variables>
dataset['GarageCond'].value_counts()
target = data_train['Survived'].values
select_features = ['AgeBinCode', 'Embarked', 'FareBinCode', 'Parch', 'Pclass', 'SibSp', 'CabinCat', 'TitleCat', 'FamilySize', 'IsAlone', 'FamilySurvival', 'NameLengthBinCode', 'female', 'male', 'TicketNum', 'TicketCode', 'CabinNum', 'child', 'female_adult', 'male_adult']<normalization>
dataset['GarageFinish']=dataset['GarageFinish'].fillna('Unf')
 dataset['GarageCond']=dataset['GarageCond'].fillna('TA')
 test['GarageFinish']=dataset['GarageFinish'].fillna('Unf')
 test['GarageCond']=dataset['GarageCond'].fillna('TA' )
scaler = StandardScaler()
df_scaled = scaler.fit_transform(df[select_features])
 train = df_scaled[0:891].copy()
test = df_scaled[891:].copy()<import_modules>
cols_null=dataset.isna().sum(axis=0)
 cols_null=cols_null[cols_null>0]
from sklearn.feature_selection import SelectKBest, f_classif<train_model>
cols_null.sort_values(ascending=False )
selector = SelectKBest(f_classif, len(select_features))
selector.fit(train, target)
 scores = -np.log10(selector.pvalues_)
 indices = np.argsort(scores)[::-1]
print('Features importance:')
 for i in range(len(scores)) :
print('%.2f %s' %(scores[indices[i]], select_features[indices[i]]))<choose_model_class>
del dataset['PoolQC']
del test['PoolQC']
del dataset['MiscFeature']
del test['MiscFeature']
del dataset['Alley']
del test['Alley']
del dataset['Fence']
del test['Fence']
kf = KFold(n_splits = 5, random_state = 1 )<train_on_grid>
dataset[['FireplaceQu','SalePrice']].fillna('O' ).groupby('FireplaceQu' ).agg(['mean','min','max','count'] )
rfc_parameters = {'max_depth' : [5], 'n_estimators' : [500], 'min_samples_split' : [9], 'random_state' : [1], 'n_jobs' : [-1]}
rfc = RandomForestClassifier()
clf_rfc = GridSearchCV(rfc, rfc_parameters, n_jobs = -1, cv = kf, scoring = 'roc_auc')
 clf_rfc.fit(train, target )<train_on_grid>
del dataset['FireplaceQu']
del test['FireplaceQu']
rfc2_parameters = {'max_depth' : [2, 5, 8, 10, 20, 50], 'n_estimators' : [10, 50, 100, 200, 500, 1000, 2000], 'min_samples_split' : [2, 3, 5, 9, 20]}
rfc2 = RandomForestClassifier(random_state = 1, n_jobs = -1)
 clf_rfc2 = RandomizedSearchCV(rfc2, rfc2_parameters, n_jobs = -1, cv = kf, scoring = 'roc_auc')
 clf_rfc2.fit(train, target )<find_best_params>
dataset['LotFrontage']=dataset['LotFrontage'].fillna(dataset['LotFrontage'].median())
 test['LotFrontage']=test['LotFrontage'].fillna(dataset['LotFrontage'].median() )
importance = clf_rfc.best_estimator_.feature_importances_
indices = np.argsort(importance)[::-1]
print(clf_rfc.best_score_)
 print(clf_rfc.score(train, target))
print(clf_rfc.best_params_)
 print('
Feature importance:')
 for i in range(len(select_features)) :
print('%.2f %s' %(importance[indices[i]], select_features[indices[i]]))<train_on_grid>
cols_null=dataset.isna().sum(axis=0)
 cols_null=cols_null[cols_null>0]
cols_null.sort_values(ascending=False )
lr_paramaters = {'C' : [0.05, 0.1, 0.2], 'random_state' : [1]}
lr = LogisticRegression()
clf_lr = GridSearchCV(lr, lr_paramaters, n_jobs = -1, cv = kf, scoring = 'roc_auc')
 clf_lr.fit(train, target )<compute_test_metric>
dataset[['GarageQual','SalePrice']].fillna('O' ).groupby('GarageQual' ).agg(['mean','min','max','count'] )
print(clf_lr.best_score_)
 print(clf_lr.score(train, target))
print(clf_lr.best_params_ )<train_on_grid>
dataset['GarageQual']=dataset['GarageQual'].fillna('TA')
 test['GarageQual']=test['GarageQual'].fillna('TA' )
svc_paramaters = {'C' : [5.5, 6, 6.5], 'kernel' : ['linear', 'rbf'], 'gamma' : ['auto', 'scale'], 'random_state' : [1]}
svc = SVC()
clf_svc = GridSearchCV(svc, svc_paramaters, n_jobs = -1, cv = kf, scoring = 'roc_auc')
 clf_svc.fit(train, target )<compute_test_metric>
dataset['GarageType']=dataset['GarageType'].fillna('Attchd')
 test['GarageType']=test['GarageType'].fillna('Attchd' )
print(clf_svc.best_score_)
 print(clf_svc.score(train, target))
print(clf_svc.best_params_ )<train_on_grid>
cols_null=dataset.isna().sum(axis=0)
 cols_null=cols_null[cols_null>0]
cols_null.sort_values(ascending=False )
gbdt_parameters = {'subsample' : [1], 'min_samples_leaf' : [3], 'learning_rate' : [0.1], 'n_estimators' : [50], 'min_samples_split' : [2], 'max_depth' : [3], 'random_state' : [1]}
gbdt = GradientBoostingClassifier()
clf_gbdt = GridSearchCV(gbdt, gbdt_parameters, n_jobs = -1, cv = kf, scoring = 'roc_auc')
 clf_gbdt.fit(train, target )<find_best_score>
dataset[['BsmtFinType2','SalePrice']].fillna('O' ).groupby('BsmtFinType2' ).agg(['mean','median','min','max','count'] )
print(clf_gbdt.best_score_)
 print(clf_gbdt.score(train, target))
print(clf_gbdt.best_params_ )<train_on_grid>
dataset['BsmtFinType2']=dataset['BsmtFinType2'].fillna('Unf')
 test['BsmtFinType2']=test['BsmtFinType2'].fillna('Unf' )
xgb_paramaters = {'subsample' : [0.7], 'min_child_weight' : [1], 'max_depth' : [3], 'learning_rate' : [0.1], 'n_estimators' : [100], 'n_jobs' : [-1], 'random_state' : [1]}
xgb = XGBClassifier()
clf_xgb = GridSearchCV(xgb, xgb_paramaters, n_jobs = -1, cv = kf, scoring = 'roc_auc')
 clf_xgb.fit(train, target )<compute_test_metric>
dataset[['BsmtExposure','SalePrice']].fillna('No' ).groupby('BsmtExposure' ).agg(['mean','median','min','max','count'] )
print(clf_xgb.best_score_)
 print(clf_xgb.score(train, target))
print(clf_xgb.best_params_ )<predict_on_test>
dataset['BsmtExposure']=dataset['BsmtExposure'].fillna('No')
 test['BsmtExposure']=test['BsmtExposure'].fillna('No' )
prediction = clf_rfc2.predict(test )<save_to_csv>
cols_null=dataset.isna().sum(axis=0)
 cols_null=cols_null[cols_null>0]
cols_null.sort_values(ascending=False )
submission = pd.DataFrame({'Survived' : prediction}, index = data_test.PassengerId)
 submission.to_csv('submission.csv', index_label = ['PassengerId'] )<import_modules>
dataset[['BsmtFinType1','SalePrice']].fillna('O' ).groupby('BsmtFinType1' ).agg(['mean','median','min','max','count'] )
print('All Good!' )<load_from_csv>
dataset['BsmtFinType1']=dataset['BsmtFinType1'].fillna('Unf')
 test['BsmtFinType1']=test['BsmtFinType1'].fillna('Unf' )
def read_data(train = 'train', test = 'test'):

 train = pd.read_csv(".. /input/" + train + ".csv")
 test = pd.read_csv(".. /input/" + test + ".csv")
 return train, test
def write_df(df, filename):

 df.to_csv(filename + '.csv', index = False)
 print(filename, 'written to csv.')
 print('All Good!' )<split>
cols_null=dataset.isna().sum(axis=0)
 cols_null=cols_null[cols_null>0]
cols_null.sort_values(ascending=False )
train, test = read_data()
print('All Good!' )<train_model>
dataset[['BsmtCond','SalePrice']].fillna('O' ).groupby('BsmtCond' ).agg(['mean','median','min','max','count'] )
print('Train data:', train.columns.values)
 print('
')
 print('Test data:', test.columns.values )<concatenate>
dataset['BsmtCond']=dataset['BsmtCond'].fillna('TA')
 test['BsmtCond']=test['BsmtCond'].fillna('TA' )
predictor_cols = test.columns.values
data = pd.concat([train[predictor_cols], test])
 data.is_copy = False
print('All Good!' )<drop_column>
cols_null=dataset.isna().sum(axis=0)
 cols_null=cols_null[cols_null>0]
cols_null.sort_values(ascending=False )
data['Travelling_alone'] = data['SibSp'] + data['Parch']
data['Travelling_alone'] = np.where(data['Travelling_alone'] > 0, 0, 1)
 data.drop(['SibSp', 'Parch'], axis=1, inplace=True)
 print('All Good!' )<drop_column>
dataset[['BsmtQual','SalePrice']].fillna('O' ).groupby('BsmtQual' ).agg(['mean','median','min','max','count'] )
data.drop(['PassengerId', 'Ticket', 'Name'], axis=1, inplace=True)
 print('All Good!' )<count_missing_values>
dataset['BsmtQual']=dataset['BsmtQual'].fillna('Gd')
 test['BsmtQual']=test['BsmtQual'].fillna('Gd' )
def na_count(data):

 na_cols = []
for i in data.columns:
if data[i].isnull().sum() != 0:
na_cols.append([i, data[i].isnull().sum() , round(data[i].isnull().sum() / len(data[i])* 100, 4)])
 return na_cols
print('All Good!' )<count_values>
cols_null=dataset.isna().sum(axis=0)
 cols_null=cols_null[cols_null>0]
cols_null.sort_values(ascending=False )
na_cols = na_count(data)
 print('Missing Values:')
 for i in na_cols:
print(i )<data_type_conversions>
test['MasVnrArea']=test['MasVnrArea'].fillna(test['MasVnrArea'].median())
 dataset['MasVnrArea']=dataset['MasVnrArea'].fillna(dataset['MasVnrArea'].median() )
data['Age'].fillna(data['Age'].median(skipna=True), inplace=True)
 print('All Good!' )<data_type_conversions>
dataset['MasVnrType'].value_counts()
data['Fare'].fillna(data['Fare'].median(skipna=True), inplace=True)
 print('All Good!' )<drop_column>
dataset['MasVnrType']=dataset['MasVnrType'].fillna('None')
 test['MasVnrType']=test['MasVnrType'].fillna('None' )
data.drop('Cabin', axis=1, inplace = True)
 print('All Good!' )<train_model>
dataset[['Electrical','SalePrice']].groupby('Electrical' ).agg(['mean','median','min','max','count'] )
data['Embarked'].fillna('S', inplace = True)
 print('All Good!' )<count_values>
dataset['Electrical']=dataset['Electrical'].fillna('SBrkr')
 test['Electrical']=test['Electrical'].fillna('SBrkr' )
na_cols = na_count(data)
 print('Missing Values:')
 for i in na_cols:
print(i )<filter>
dataset=dataset.loc[dataset['LotFrontage']<300,:]
data[data['Fare'] == max(data['Fare'])]<categorify>
dataset=dataset.reset_index(drop=True )
data = pd.get_dummies(data, columns=['Pclass'])
 print('All Good!' )<drop_column>
dataset=dataset.loc[dataset['LotArea']<dataset['LotArea'].quantile(0.995),:]
dataset=dataset.reset_index(drop=True )
data.drop('Pclass_3', axis=1, inplace=True)
 print('All Good!' )<categorify>
dataset=dataset.loc[dataset['MasVnrArea']<=dataset['MasVnrArea'].quantile(0.999),:]
dataset=dataset.reset_index(drop=True )
data = pd.get_dummies(data, columns=['Sex'])
 print('All Good!' )<drop_column>
dataset=dataset.loc[dataset['BsmtFinSF2']<=dataset['BsmtFinSF2'].quantile(0.9991),:]
dataset=dataset.reset_index(drop=True )
data.drop('Sex_female', axis=1, inplace=True)
 print('All Good!' )<categorify>
dataset=dataset.loc[dataset['EnclosedPorch']<=dataset['EnclosedPorch'].quantile(0.997),:]
dataset=dataset.reset_index(drop=True )
data = pd.get_dummies(data, columns=['Embarked'])
 print('All Good!' )<drop_column>
cols_null=test.isna().sum(axis=0)
 cols_null=cols_null[cols_null>0]
cols_null.sort_values(ascending=False )
data.drop('Embarked_Q', axis=1, inplace=True)
 print('All Good!' )<feature_engineering>
from sklearn.preprocessing import StandardScaler
data['is_minor'] = np.where(data['Age'] <= 16, 1, 0 )<split>
def _check_numberic(data):
return pd.to_numeric(data[data.notna() ], errors='coerce' ).notnull().all()
train_df = data.iloc[:891]
test_df = data.iloc[891:]
print('All Good!' )<split>
from sklearn.preprocessing import LabelEncoder
X_train, X_test, Y_train, Y_test = train_test_split(train_df, train['Survived'], test_size = 0.33, random_state = 0)
 print('All Good!' )<train_model>
test.isna().sum()
DTC3 = tree.DecisionTreeClassifier(max_depth = 3)
 DTC3.fit(X_train, Y_train)
 Y_pred_DTC3 = DTC3.predict(X_test)
 print('All Good!' )<compute_test_metric>
newdata=pd.concat([dataset,test],axis=0 )
DTC3_mae = mean_absolute_error(Y_test, Y_pred_DTC3)
 print('Decision Tree Classifier with max_depth = 3:
Mean Absolute Error: ', DTC3_mae )<predict_on_test>
newdata.SalePrice.isna().sum()
DTC3_pred = DTC3.predict(test_df)
 print('All Good!' )<save_to_csv>
newdata=newdata.reset_index(drop=True )
DTC3_submission = pd.DataFrame({'PassengerId':test['PassengerId'], 'Survived':DTC3_pred})
 DTC3_submission.to_csv('DTC3_submission.csv', index=False)
 print('All Good!' )<train_model>
del newdata['3SsnPorch']
del newdata['Condition2']
del newdata['GarageCars']
del newdata['LowQualFinSF']
del newdata['MiscVal']
del newdata['RoofMatl']
del newdata['Street']
del newdata['Utilities']
RFC = RandomForestClassifier(n_estimators = 100)
 RFC.fit(X_train, Y_train)
 Y_pred_RFC = RFC.predict(X_test)
 print('All Good!' )<compute_test_metric>
newdata['HalfBath']=[1 if val>0 else 0 for val in newdata['HalfBath']]
newdata['PoolArea']=[1 if val>0 else 0 for val in newdata['PoolArea']]
RFC_mae = mean_absolute_error(Y_test, Y_pred_RFC)
 print('Random Forest Classifier with max_depth = 3:
Mean Absolute Error: ', RFC_mae )<predict_on_test>
nacols=newdata.isna().sum()
RFC_pred = RFC.predict(test_df)
 print('All Good!' )<save_to_csv>
nacols[nacols>0]
RFC_submission = pd.DataFrame({'PassengerId':test['PassengerId'], 'Survived':RFC_pred})
 RFC_submission.to_csv('RFC_submission.csv', index=False)
 print('All Good!' )<train_model>
newdata['BsmtFinSF1']=newdata['BsmtFinSF1'].fillna(newdata['BsmtFinSF1'].median())
 newdata['BsmtFinSF2']=newdata['BsmtFinSF2'].fillna(newdata['BsmtFinSF2'].median())
 newdata['BsmtFullBath']=newdata['BsmtFullBath'].fillna(newdata['BsmtFullBath'].median())
 newdata['BsmtHalfBath']=newdata['BsmtHalfBath'].fillna(newdata['BsmtHalfBath'].median())
 newdata['BsmtUnfSF']=newdata['BsmtUnfSF'].fillna(newdata['BsmtUnfSF'].median())
 newdata['GarageArea']=newdata['GarageArea'].fillna(newdata['GarageArea'].median())
 newdata['TotalBsmtSF']=newdata['TotalBsmtSF'].fillna(newdata['TotalBsmtSF'].median() )
LogReg = LogisticRegression()
LogReg.fit(X_train, Y_train)
 Y_pred_LogReg = LogReg.predict(X_test)
 print('All Good!' )<compute_test_metric>
col='Exterior1st'
newdata[col]=newdata[col].fillna(newdata[col].value_counts().idxmax())
 col='Exterior2nd'
newdata[col]=newdata[col].fillna(newdata[col].value_counts().idxmax())
 col='Functional'
newdata[col]=newdata[col].fillna(newdata[col].value_counts().idxmax())
 col='KitchenQual'
newdata[col]=newdata[col].fillna(newdata[col].value_counts().idxmax())
 col='MSZoning'
newdata[col]=newdata[col].fillna(newdata[col].value_counts().idxmax())
 col='SaleType'
newdata[col]=newdata[col].fillna(newdata[col].value_counts().idxmax() )
LogReg_mae = mean_absolute_error(Y_test, Y_pred_LogReg)
 print('Random Forest Classifier with max_depth = 3:
Mean Absolute Error: ', LogReg_mae )<predict_on_test>
na=newdata.isna().sum()
LogReg_pred = LogReg.predict(test_df)
 print('All Good!' )<save_to_csv>
na[na>0]
LogReg_submission = pd.DataFrame({'PassengerId':test['PassengerId'], 'Survived':LogReg_pred})
 LogReg_submission.to_csv('LogReg_submission.csv', index=False)
 print('All Good!' )<set_options>
le=LabelEncoder()
for col in newdata.columns:
if not _check_numberic(newdata[col]):
le.fit(newdata[col])
 newdata[col]=le.transform(newdata[col] )
sns.set(style="darkgrid")
 warnings.filterwarnings('ignore')
 SEED = 42<load_from_csv>
test_data=newdata.loc[newdata.SalePrice.isna() ,:]
train=pd.read_csv("/kaggle/input/titanic/train.csv")
 test=pd.read_csv("/kaggle/input/titanic/test.csv")
 titanic = train.append(test, ignore_index=True)
 train_idx = len(train)
 test_idx = len(titanic)- len(test)
 passengerId = test.PassengerId<count_missing_values>
train_data=newdata.loc[newdata.SalePrice.notna() ,:]
titanic.isnull().sum() [titanic.isnull().sum() >0]<count_values>
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
titanic['Title'] = titanic.Name.apply(lambda name: name.split(',')[1].split('.')[0].strip())
 print(titanic.Title.value_counts() )<define_variables>
from sklearn.model_selection import train_test_split
standardized_titles = {
"Capt": "Officer",
"Col": "Officer",
"Major": "Officer",
"Jonkheer": "Aristorcat",
"Don": "Aristorcat",
"Sir" : "Aristorcat",
"Dr": "Officer",
"Rev": "Officer",
"the Countess":"Aristorcat",
"Dona": "Aristorcat",
"Mme": "Mrs",
"Mlle": "Miss",
"Ms": "Mrs",
"Mr" : "Mr",
"Mrs" : "Mrs",
"Miss" : "Miss",
"Master" : "Master",
"Lady" : "Aristorcat"
}
titanic.Title = titanic.Title.map(standardized_titles)
 print(titanic.Title.value_counts() )<feature_engineering>
from sklearn.model_selection import train_test_split
titanic.Age = grouped.Age.apply(lambda x: x.fillna(x.median()))<data_type_conversions>
features.remove('Id')
 features.remove('SalePrice' )
titanic.Embarked = titanic.Embarked.fillna(titanic.Embarked.mode() [0] )<categorify>
train=train_data[features].values
train_y=train_data['SalePrice']
med_fare = titanic.groupby(['Pclass', 'Parch', 'SibSp'] ).Fare.median() [3][0][0]
titanic['Fare'] = titanic['Fare'].fillna(med_fare )<feature_engineering>
sc= RobustScaler()
sc.fit(train)
 train=sc.transform(train )
titanic['Cabin'] = titanic['Cabin'].fillna('U')
 titanic["Cabin"] = pd.Series([i[0] if not pd.isnull(i)else 'X' for i in titanic['Cabin'] ] )<count_missing_values>
from sklearn.model_selection import train_test_split
titanic.isnull().sum() [titanic.isnull().sum() >0]<groupby>
from sklearn.model_selection import train_test_split
grouped = titanic.groupby(['Sex'])
 grouped.Survived.mean()<groupby>
x_train,x_test,y_train,y_test=train_test_split(train,train_y,test_size=0.2 )
grouped = titanic.groupby(['Pclass'])
 grouped.Survived.mean()<groupby>
rf = RandomForestRegressor(n_estimators = 120, random_state = 42 )
grouped = titanic.groupby(['SibSp'])
 grouped.Survived.mean()<groupby>
rf.fit(x_train, y_train )
grouped = titanic.groupby(['Parch'])
 grouped.Survived.mean()<groupby>
predictions=rf.predict(x_test)
 rmsle(y_test.to_list() ,predictions )
grouped = titanic.groupby(['Cabin'])
 grouped.Survived.mean()<groupby>
from sklearn.metrics import mean_squared_error
grouped = train.groupby(['Embarked'])
 grouped.Survived.mean()<groupby>
from sklearn.metrics import mean_squared_error
titanic['Ticket_Frequency'] = titanic.groupby('Ticket')['Ticket'].transform('count')
 grouped = titanic.groupby(['Ticket_Frequency'])
 grouped.Survived.mean()<feature_engineering>
import xgboost as xgb
titanic['Fare'] =(titanic['Fare'] / titanic['Ticket_Frequency'] )<string_transform>
model=xgb.XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.4603, gamma=0.0468,
importance_type='split', learning_rate=0.05, max_delta_step=0,
max_depth=20, min_child_weight=1.7817, missing=None,
n_estimators=1024, n_jobs=1, nthread=-1, objective='reg:squarederror',
random_state=7, reg_alpha=0.464, reg_lambda=0.8571,
scale_pos_weight=1, seed=None, silent=1, subsample=0.5213,
verbosity=1 )
def extract_surname(data):
families = []
for i in range(len(data)) :
name = data.iloc[i]
if '(' in name:
name_no_bracket = name.split('(')[0]
else:
name_no_bracket = name
family = name_no_bracket.split(',')[0]
title = name_no_bracket.split(',')[1].strip().split(' ')[0]
for c in string.punctuation:
family = family.replace(c, '' ).strip()
families.append(family)
 return families
titanic['Family'] = extract_surname(titanic['Name'])
 df_train = titanic.loc[:890]
df_test = titanic.loc[891:]
dfs = [df_train, df_test]<define_variables>
model.fit(x_train,y_train )
mean_survival_rate = np.mean(df_train['Survived'])
 train_family_survival_rate = []
train_family_survival_rate_NA = []
test_family_survival_rate = []
test_family_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Family'][i] in family_rates:
train_family_survival_rate.append(family_rates[df_train['Family'][i]])
 train_family_survival_rate_NA.append(1)
 else:
train_family_survival_rate.append(mean_survival_rate)
 train_family_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Family'].iloc[i] in family_rates:
test_family_survival_rate.append(family_rates[df_test['Family'].iloc[i]])
 test_family_survival_rate_NA.append(1)
 else:
test_family_survival_rate.append(mean_survival_rate)
 test_family_survival_rate_NA.append(0)
 df_train['Family_Survival_Rate'] = train_family_survival_rate
df_train['Family_Survival_Rate_NA'] = train_family_survival_rate_NA
df_test['Family_Survival_Rate'] = test_family_survival_rate
df_test['Family_Survival_Rate_NA'] = test_family_survival_rate_NA
train_ticket_survival_rate = []
train_ticket_survival_rate_NA = []
test_ticket_survival_rate = []
test_ticket_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Ticket'][i] in ticket_rates:
train_ticket_survival_rate.append(ticket_rates[df_train['Ticket'][i]])
 train_ticket_survival_rate_NA.append(1)
 else:
train_ticket_survival_rate.append(mean_survival_rate)
 train_ticket_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Ticket'].iloc[i] in ticket_rates:
test_ticket_survival_rate.append(ticket_rates[df_test['Ticket'].iloc[i]])
 test_ticket_survival_rate_NA.append(1)
 else:
test_ticket_survival_rate.append(mean_survival_rate)
 test_ticket_survival_rate_NA.append(0)
 df_train['Ticket_Survival_Rate'] = train_ticket_survival_rate
df_train['Ticket_Survival_Rate_NA'] = train_ticket_survival_rate_NA
df_test['Ticket_Survival_Rate'] = test_ticket_survival_rate
df_test['Ticket_Survival_Rate_NA'] = test_ticket_survival_rate_NA<feature_engineering>
predictions=model.predict(x_test)
 rmsle(y_test.to_list() ,predictions )
for df in [df_train, df_test]:
df['Survival_Rate'] =(df['Ticket_Survival_Rate'] + df['Family_Survival_Rate'])/ 2
df['Survival_Rate_NA'] =(df['Ticket_Survival_Rate_NA'] + df['Family_Survival_Rate_NA'])/ 2<categorify>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
non_numeric_features = ['Embarked', 'Sex', 'Deck', 'Title', 'Family_Size_Grouped', 'Age', 'Fare', 'NameLength']
for df in dfs:
for feature in non_numeric_features:
df[feature] = LabelEncoder().fit_transform(df[feature] )<categorify>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train)
 rmse= np.sqrt(-cross_val_score(model, train, train_y, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
cat_features = ['Pclass', 'Sex', 'Deck', 'Embarked', 'Title', 'Family_Size_Grouped']
encoded_features = []
for df in dfs:
for feature in cat_features:
encoded_feat = OneHotEncoder().fit_transform(df[feature].values.reshape(-1, 1)).toarray()
n = df[feature].nunique()
cols = ['{}_{}'.format(feature, n)for n in range(1, n + 1)]
encoded_df = pd.DataFrame(encoded_feat, columns=cols)
 encoded_df.index = df.index
encoded_features.append(encoded_df)
 df_train = pd.concat([df_train, *encoded_features[:6]], axis=1)
 df_test = pd.concat([df_test, *encoded_features[6:]], axis=1 )<concatenate>
score=rmsle_cv(model_lgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
titanic = pd.concat([df_train, df_test], sort=True ).reset_index(drop=True)
 drop_cols = ['Deck', 'Cabin', 'Embarked', 'Family', 'Family_Size', 'Family_Size_Grouped', 'Survived',
'Name', 'Parch', 'PassengerId', 'Pclass', 'Sex', 'SibSp', 'Ticket', 'Title',
'Ticket_Survival_Rate', 'Family_Survival_Rate', 'Ticket_Survival_Rate_NA', 'Family_Survival_Rate_NA']
titanic.drop(columns=drop_cols, inplace=True)
 titanic.head()<normalization>
model_lgb.fit(train,train_y )
X_train = StandardScaler().fit_transform(df_train.drop(columns=drop_cols))
y_train = df_train['Survived'].values
X_test = StandardScaler().fit_transform(df_test.drop(columns=drop_cols))
print('X_train shape: {}'.format(X_train.shape))
print('y_train shape: {}'.format(y_train.shape))
print('X_test shape: {}'.format(X_test.shape))<choose_model_class>
model.fit(x_train,y_train)
 predictions=model.predict(x_test)
 rmsle(y_test.to_list() ,predictions )
max_evals = 2
space={
'n_estimators': hp.uniform("n_estimators", 500, 2000),
'max_features': hp.choice("max_features", ["auto", "sqrt"]),
'max_depth': hp.uniform("max_depth", 2, 12),
'min_samples_split': hp.uniform("min_samples_split", 2, 11),
'min_samples_leaf': hp.uniform("min_samples_leaf", 2, 11),
}
def hyperparameter_tuning(space):
model=RandomForestClassifier(
 n_estimators =int(space['n_estimators']),
max_depth =int(space['max_depth']),
max_features = space['max_features'],
min_samples_split =int(space['min_samples_split']),
min_samples_leaf =int(space['min_samples_leaf']),
random_state = SEED 
)
 scores = cross_val_score(model, X_train, y_train, scoring='accuracy', cv=StratifiedKFold(n_splits=5,shuffle=True))
print("SCORE: %0.2f" %(-scores.mean()))
return {'loss':-scores.mean() , 'status': STATUS_OK }
trials = Trials()
best = fmin(fn=hyperparameter_tuning,
space=space,
algo=tpe.suggest,
max_evals=max_evals,
trials=trials)
 print(best )<choose_model_class>
model_lgb=lgb.LGBMRegressor(bagging_fraction=0.8, bagging_freq=5, bagging_seed=9,
boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,
feature_fraction=0.8, feature_fraction_seed=9,
importance_type='gain', learning_rate=0.02, max_bin=55,
max_depth=-1, min_child_samples=30, min_child_weight=0.01,
min_data_in_leaf=6, min_split_gain=0.0,
min_sum_hessian_in_leaf=11, n_estimators=1024, n_jobs=-1,
num_leaves=20, objective='regression', random_state=None,
reg_alpha=0.01, reg_lambda=0.01, silent=True, subsample=1.0,
subsample_for_bin=200000, subsample_freq=0 )
model = RandomForestClassifier(criterion='gini',
n_estimators=1750,
max_depth=7,
min_samples_split=6,
min_samples_leaf=6,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=1 )<train_model>
model_lgb.fit(x_train,y_train)
 predictions=model_lgb.predict(x_test)
 rmsle(y_test.to_list() ,predictions )
N = 5
oob = 0
probs = pd.DataFrame(np.zeros(( len(X_test), N * 2)) , columns=['Fold_{}_Prob_{}'.format(i, j)for i in range(1, N + 1)for j in range(2)])
 importances = pd.DataFrame(np.zeros(( X_train.shape[1], N)) , columns=['Fold_{}'.format(i)for i in range(1, N + 1)], index=titanic.columns)
 skf = StratifiedKFold(n_splits=N, random_state=N, shuffle=True)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(X_train, y_train), 1):
print('Fold {}
'.format(fold))
model.fit(X_train[trn_idx], y_train[trn_idx])
 probs.loc[:, 'Fold_{}_Prob_0'.format(fold)] = model.predict_proba(X_test)[:, 0]
probs.loc[:, 'Fold_{}_Prob_1'.format(fold)] = model.predict_proba(X_test)[:, 1]
importances.iloc[:, fold - 1] = model.feature_importances_
oob += model.oob_score_ / N
print('Fold {} OOB Score: {}
'.format(fold, model.oob_score_))
print('Average OOB Score: {}'.format(oob))<feature_engineering>
test_sc=sc.transform(test_data[features].values )
class_survived = [col for col in probs.columns if col.endswith('Prob_1')]
probs['1'] = probs[class_survived].sum(axis=1)/ N
probs['0'] = probs.drop(columns=class_survived ).sum(axis=1)/ N
probs['pred'] = 0
pos = probs[probs['1'] >= 0.50].index
probs.loc[pos, 'pred'] = 1
y_pred = probs['pred'].astype(int)
 submission_df = pd.DataFrame(columns=['PassengerId', 'Survived'])
 submission_df['PassengerId'] = df_test['PassengerId']
submission_df['Survived'] = y_pred.values<save_to_csv>
model_lgb.fit(train,train_y)
 predicted_values=model_lgb.predict(test_sc )
os.chdir('/kaggle/working')
 submission_df.to_csv('titanic_submissions.csv', header=True, index=False )<set_options>
model.fit(train,train_y)
 predicted_values2=model.predict(test_sc )
warnings.filterwarnings('ignore')
 sns.set(rc={'figure.figsize':(15,12)})
 %matplotlib inline<load_from_csv>
output=pd.DataFrame()
output['Id']=test['Id']
output['SalePrice']=predicted_values
train_df = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test_df = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )<drop_column>
output.to_csv('sample_submission.csv',index=False )