Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<count_values>
warnings.filterwarnings("ignore")
 plt.style.use('ggplot')
train_df['child'] = [1 if i<16 else 0 for i in train_df.Age]
test_df['child'] = [1 if i<16 else 0 for i in test_df.Age]
train_df.child.value_counts()<feature_engineering>
df_train=pd.read_csv('.. /input/train.csv', index_col='Id')
 df_test=pd.read_csv('.. /input/test.csv', index_col='Id')
 train_ID = df_train.index
test_ID = df_test.index
train_df['calculated_fare'] = train_df.Fare/train_df.family_size
test_df['calculated_fare'] = test_df.Fare/test_df.family_size 
<set_options>
df_train = df_train.drop(( df_train[df_train['GrLivArea']>4000] ).index )
train_df.calculated_fare.mode()<categorify>
quantitative = [i for i in df_train.columns if df_train.dtypes[i] != 'object']
quantitative.remove('SalePrice')
 qualitative = [j for j in df_train.columns if df_train.dtypes[j] == 'object']
print('Quantitative:',quantitative)
 print('')
 print('Qualitative:',qualitative )
train_df['fare_group'] = train_df['calculated_fare'].map(fare_group)
 test_df['fare_group'] = test_df['calculated_fare'].map(fare_group )<categorify>
merged_data = df_train.append(df_test, sort=False ).reset_index(drop=True)
 print("The size of the merged data is:", merged_data.shape)
 ntrain = df_train.shape[0]
ntest = df_test.shape[0]
y_train = df_train.SalePrice.values
train_df = pd.get_dummies(train_df, columns=['Title',"Pclass",'Embarked', 'family_group', 'fare_group'], drop_first=True)
 test_df = pd.get_dummies(test_df, columns=['Title',"Pclass",'Embarked', 'family_group', 'fare_group'], drop_first=True)
 train_df.drop(['Cabin', 'family_size','Ticket','Name', 'Fare'], axis=1, inplace=True)
 test_df.drop(['Ticket','Name','family_size',"Fare",'Cabin'], axis=1, inplace=True)
<set_options>
merged_data["Alley"] = merged_data["Alley"].fillna("None")
 merged_data["Fence"] = merged_data["Fence"].fillna("None")
 merged_data["MiscFeature"] = merged_data["MiscFeature"].fillna("None")
 merged_data["PoolQC"] = merged_data["PoolQC"].fillna("None")
 merged_data["FireplaceQu"] = merged_data["FireplaceQu"].fillna("None")
 merged_data["LotFrontage"] = merged_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
merged_data[col] = merged_data[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
merged_data[col] = merged_data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
merged_data[col] = merged_data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
merged_data[col] = merged_data[col].fillna('None')
 merged_data["MasVnrType"] = merged_data["MasVnrType"].fillna("None")
 merged_data["MasVnrArea"] = merged_data["MasVnrArea"].fillna(0)
 merged_data['MSZoning'] = merged_data['MSZoning'].fillna(merged_data['MSZoning'].mode() [0])
 merged_data = merged_data.drop(['Utilities'], axis=1)
 merged_data["Functional"] = merged_data["Functional"].fillna("Typ")
 merged_data['Electrical'] = merged_data['Electrical'].fillna(merged_data['Electrical'].mode() [0])
 merged_data['KitchenQual'] = merged_data['KitchenQual'].fillna(merged_data['KitchenQual'].mode() [0])
 merged_data['Exterior1st'] = merged_data['Exterior1st'].fillna(merged_data['Exterior1st'].mode() [0])
 merged_data['Exterior2nd'] = merged_data['Exterior2nd'].fillna(merged_data['Exterior2nd'].mode() [0])
 merged_data['SaleType'] = merged_data['SaleType'].fillna(merged_data['SaleType'].mode() [0])
 merged_data['MSSubClass'] = merged_data['MSSubClass'].fillna("None" )
pd.options.display.max_columns = 99 
<categorify>
merged_data_NA_VALUES =merged_data.isnull().sum()
train_df['age_group'] = train_df['Age'].map(age_group_fun)
 test_df['age_group'] = test_df['Age'].map(age_group_fun )<categorify>
merged_data['TotalSF'] = merged_data['TotalBsmtSF'] + merged_data['1stFlrSF'] + merged_data['2ndFlrSF']
train_df = pd.get_dummies(train_df,columns=['age_group'], drop_first=True)
 test_df = pd.get_dummies(test_df,columns=['age_group'], drop_first=True)
 train_df.drop(['Age','calculated_fare'],axis=1,inplace=True)
 test_df.drop(['Age','calculated_fare'],axis=1,inplace=True )<categorify>
merged_data['MSSubClass'] = merged_data['MSSubClass'].apply(str)
 merged_data['OverallCond'] = merged_data['OverallCond'].astype(str)
 merged_data['YrSold'] = merged_data['YrSold'].astype(str)
 merged_data['MoSold'] = merged_data['MoSold'].astype(str )
train_df.head()
train_df.drop(['Title_Rev','age_group_old','age_group_teenager','age_group_senior_citizen','Embarked_Q'],axis=1,inplace=True)
 test_df.drop(['Title_Rev','age_group_old','age_group_teenager','age_group_senior_citizen','Embarked_Q'],axis=1,inplace=True )<prepare_x_and_y>
columns = ['LandContour', 'MSZoning', 'Alley',
'MasVnrType', 'ExterQual', 'ExterCond',
'BsmtQual', 'BsmtCond', 'BsmtExposure',
'BsmtFinType1', 'BsmtFinType2', 'HeatingQC',
'CentralAir', 'KitchenQual', 'FireplaceQu',
'GarageFinish', 'GarageQual', 'GarageCond',
'PavedDrive', 'PoolQC', 'MiscFeature']
for col in columns:
lbl = LabelEncoder()
lbl.fit(list(merged_data[col].values))
merged_data[col] = lbl.transform(list(merged_data[col].values))
columns_Qual=['LotShape', 'LotConfig', 'LandSlope', 'Neighborhood',
'Condition1', 'Condition2', 'BldgType', 'HouseStyle',
'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd',
'Foundation', 'Heating', 'Electrical', 'Functional',
'GarageType', 'GarageCond', 'Fence', 'SaleType',
'SaleCondition', 'Street']
temp = pd.get_dummies(merged_data[columns_Qual], drop_first=True)
 merged_data = merged_data.drop(columns_Qual, axis=1)
 merged_data = pd.concat([merged_data, temp], axis=1 )
X = train_df.drop('Survived', 1)
 y = train_df['Survived'] 
<choose_model_class>
df_train = merged_data[merged_data['SalePrice'].notnull() ]
df_test = merged_data[merged_data['SalePrice'].isnull() ].drop('SalePrice', axis=1 )
classifiers = [
KNeighborsClassifier(3),
svm.SVC(probability=True),
DecisionTreeClassifier() ,
CatBoostClassifier() ,
XGBClassifier() ,
RandomForestClassifier() ,
AdaBoostClassifier() ,
GradientBoostingClassifier() ,
GaussianNB() ,
LinearDiscriminantAnalysis() ,
QuadraticDiscriminantAnalysis() ,
LogisticRegression() ]
log_cols = ["Classifier", "Accuracy"]
log= pd.DataFrame(columns=log_cols)
<train_model>
x_train = df_train.drop(['SalePrice'], axis=1)
 y_train = df_train['SalePrice']
x_test = df_test
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split,StratifiedShuffleSplit 
<import_modules>
scaler = preprocessing.RobustScaler() ;
x_train_scaled = pd.DataFrame(scaler.fit_transform(x_train), columns=x_train.columns)
 x_test_scaled = pd.DataFrame(scaler.transform(x_test), columns=x_test.columns )
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import cross_val_score, StratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import BaggingClassifier
from sklearn.metrics import accuracy_score,classification_report, precision_recall_curve, confusion_matrix<normalization>
KRR = KernelRidge(alpha=0.05, kernel='polynomial', degree=1, coef0=2.5 )
std_scaler = StandardScaler()
X = std_scaler.fit_transform(X)
 testframe = std_scaler.fit_transform(test_df)
 testframe.shape 
<split>
lasso = linear_model.Lasso(alpha=0.001, max_iter=5000, random_state=42 )
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.20,random_state=1000 )<predict_on_test>
GBoost = ensemble.GradientBoostingRegressor(n_estimators=1000, learning_rate=0.05, max_depth=3, max_features='sqrt', loss='huber', random_state=42 )
logreg = LogisticRegression(solver='liblinear', penalty='l1')
 logreg.fit(X_train,y_train)
 predict=logreg.predict(X_test)
 print(accuracy_score(y_test,predict))
print(confusion_matrix(y_test,predict))
print(precision_score(y_test,predict))
print(recall_score(y_test,predict))<define_search_space>
class Averaging_the_models(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, cool_models, peso):
self.cool_models = cool_models
self.peso = peso
def fit(self, X, y):
self.cool_models_ = [clone(x)for x in self.cool_models]
for model in self.cool_models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([(model.predict(X)* peso)for model, peso in zip(self.cool_models_, self.peso)])
 return np.sum(predictions, axis=1 )
C_vals = [0.0001, 0.001, 0.01, 0.1,0.13,0.2,.15,.25,.275,.33, 0.5,.66, 0.75, 1.0, 2.5, 4.0,4.5,5.0,5.1,5.5,6.0, 10.0, 100.0, 1000.0]
penalties = ['l1','l2']
param = {'penalty': penalties, 'C': C_vals, }
grid = GridSearchCV(logreg, param,verbose=False, cv = StratifiedKFold(n_splits=5,random_state=10,shuffle=True), n_jobs=1,scoring='accuracy' )<train_model>
regression = Averaging_the_models(cool_models=(KRR, lasso, GBoost), peso=[0.25, 0.25, 0.50] )
grid.fit(X_train,y_train)
 print(grid.best_params_)
 print(grid.best_score_)
 print(grid.best_estimator_ )<compute_train_metric>
regression.fit(x_train_scaled, np.log1p(y_train))
result = np.expm1(regression.predict(x_test_scaled))
logreg_grid = LogisticRegression(penalty=grid.best_params_['penalty'], C=grid.best_params_['C'])
 logreg_grid.fit(X_train,y_train)
 y_pred = logreg_grid.predict(X_test)
 logreg_accy = round(accuracy_score(y_test, y_pred), 3)
 print(logreg_accy)
 print(confusion_matrix(y_test,y_pred))
print(precision_score(y_test,y_pred))
print(recall_score(y_test,y_pred))<compute_train_metric>
subFDS = pd.DataFrame({
"Id": test_ID,
"SalePrice": result
})
 subFDS.to_csv("subFDS.csv", index=False )
<define_search_space><EOS> 
def rmse_cv(model, x, y):
rmse = np.sqrt(-cross_val_score(model, x, y, scoring="neg_mean_squared_error", cv=5))
return rmse
score = rmse_cv(regression, x_train_scaled, np.log1p(y_train))
print(round(score.mean() , 5))