Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<normalization>
RANDOM_SEED = 42
CV_FOLDS = 12
sc = StandardScaler()
features = train[['Pclass','Sex','Name','Fare']].values
features = sc.fit_transform(features)
 target = train['Survived'].values<split>
data_types = {'MSZoning': 'category',
'Street': 'category',
'Alley': 'category',
'LotShape': 'category',
'LandContour': 'category',
'Utilities': 'category',
'LotConfig': 'category',
'LandSlope': 'category',
'Neighborhood': 'category',
'Condition1': 'category',
'Condition2': 'category',
'BldgType': 'category',
'HouseStyle': 'category',
'RoofStyle': 'category',
'RoofMatl': 'category',
'Exterior1st': 'category',
'Exterior2nd': 'category',
'MasVnrType': 'category',
'ExterQual': 'category',
'ExterCond': 'category',
'Foundation': 'category',
'BsmtQual': 'category',
'BsmtCond': 'category',
'BsmtExposure': 'category',
'BsmtFinType1': 'category',
'BsmtFinType2': 'category',
'ExterQual': 'category',
'Heating': 'category',
'HeatingQC': 'category',
'CentralAir': 'category',
'Electrical': 'category',
'KitchenQual': 'category',
'Functional': 'category',
'FireplaceQu': 'category',
'GarageType': 'category',
'GarageFinish': 'category',
'GarageQual': 'category',
'GarageCond': 'category',
'PavedDrive': 'category',
'PoolQC': 'category',
'Fence': 'category',
'MiscFeature': 'category',
'SaleType': 'category',
'SaleCondition': 'category',
'MoSold': 'int64',
'YrSold': 'int64',
'YearBuilt': 'int64'}
data_types = {'MoSold': 'int64',
'YrSold': 'int64',
'YearBuilt': 'int64'}
df_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv', index_col='Id', dtype=data_types)
 df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv', index_col='Id', dtype=data_types )
x_train, x_test, y_train, y_test = train_test_split(features, target, test_size = 0.25, random_state = 2 )<train_on_grid>
def missing_values_table(df):
mis_val = df.isnull().sum()
mis_val_percent = 100 * df.isnull().sum() / len(df)
 mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)
 mis_val_table_ren_columns = mis_val_table.rename(
 columns = {0 : 'No.Missing Values', 1 : '%'})
 mis_val_table_ren_columns = mis_val_table_ren_columns[
mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(
 '%', ascending=False ).round(1)
 print("There are " + str(mis_val_table_ren_columns.shape[0])+
" features with missing values.")
 return mis_val_table_ren_columns
missing_values_table(df_train )
def grid_search_cv(classifier,hyperparameters):
GSCV = GridSearchCV(classifier,hyperparameters,cv = 10)
 model = GSCV.fit(x_test,y_test)
 print(model)
 print('HIGHEST ACCURACY : ','{0:.2%}'.format(model.best_score_))
print('BEST PARAMETERS : ',model.best_params_ )<import_modules>
numerical_features = df_train.select_dtypes(include='number' ).columns.tolist()
numerical_features.remove('BsmtFinSF2')
 numerical_features.remove('LowQualFinSF')
 numerical_features.remove('BsmtHalfBath')
 numerical_features.remove('KitchenAbvGr')
 numerical_features.remove('EnclosedPorch')
 numerical_features.remove('3SsnPorch')
 numerical_features.remove('ScreenPorch')
 numerical_features.remove('PoolArea')
 numerical_features.remove('MiscVal')
 for feature in numerical_features:
num_summary(df_train, feature )
from sklearn.linear_model import LogisticRegression<choose_model_class>
def get_redundant_pairs(df):

 pairs_to_drop = set()
cols = df.columns
for i in range(0, df.shape[1]):
for j in range(0, i+1):
pairs_to_drop.add(( cols[i], cols[j]))
return pairs_to_drop
def get_top_abs_correlations(df, n=5):

 au_corr = df.corr().abs().unstack()
labels_to_drop = get_redundant_pairs(df)
 au_corr = au_corr.drop(labels=labels_to_drop ).sort_values(ascending=False)
 return au_corr[0:n]
print("Top Absolute Correlations")
 print("=========================
")
 print(get_top_abs_correlations(df_train[df_train.select_dtypes(include='number' ).columns.tolist() ], 15))
classifier_lr = LogisticRegression(random_state = 0,C=10,penalty= 'l2' )<compute_test_metric>
data_types = {'MoSold': 'int64',
'YrSold': 'int64',
'YearBuilt': 'int64'}
df_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv', index_col='Id', dtype=data_types)
 df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv', index_col='Id', dtype=data_types )
model_evaluation(classifier_lr )<import_modules>
for data in [df_train, df_test]:
data['haspool'] = data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 data['has2ndfloor'] = data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 data['hasgarage'] = data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 data['hasbsmt'] = data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 data['hasfireplace'] = data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
from sklearn.svm import SVC<choose_model_class>
numerical_features = df_train.select_dtypes(include='number' ).columns.tolist()
numerical_features.remove('SalePrice')
 categorical_features = df_train.select_dtypes(include='object' ).columns.tolist()
classifier_svc = SVC(kernel = 'linear',C = 0.1 )<find_best_params>
numerical_features_to_drop = ['FireplaceQu'] + ['GarageCars', 'GarageYrBlt', 'TotRmsAbvGrd', '1stFlrSF']
for data in [df_train, df_test]:
data.drop(numerical_features_to_drop,inplace=True, axis=1 )
hyperparameters = {'C' : [0.01,0.1,1,10,100]}
grid_search_cv(classifier_svc,hyperparameters )<compute_train_metric>
categorical_features_na = ['PoolQC', 'MiscFeature', 'Alley', 'Fence']
for data in [df_train, df_test]:
for feature in categorical_features_na:
data[feature].fillna("missing_value", inplace = True )
model_evaluation(classifier_svc )<import_modules>
missing_values_table(df_train )
from sklearn.tree import DecisionTreeClassifier<choose_model_class>
class TransLog(BaseEstimator, TransformerMixin):
def __init__(self, feature_index):
self.feature_index = feature_index
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X_trans = np.log(X[:, self.feature_index])
 return np.c_[X[:, 0:self.feature_index - 1], X_trans, X[:, self.feature_index:]]
classifier_dt = DecisionTreeClassifier(criterion = 'entropy' )<compute_test_metric>
y_train = np.log(df_train['SalePrice'])
 X_train = df_train
X_train.drop(columns=['SalePrice'], inplace=True)
 X_test = df_test
numerical_features = df_test.select_dtypes(include='number' ).columns.tolist()
categorical_features = df_test.select_dtypes(include='object' ).columns.tolist()
model_evaluation(classifier_dt )<import_modules>
log_feature_index = numerical_features.index('GrLivArea' )
from sklearn.ensemble import RandomForestClassifier<choose_model_class>
num_pipeline = Pipeline([ 
('simple_imp', SimpleImputer(strategy='median')) , 
('log_trans', TransLog(feature_index=log_feature_index)) , 
('std_scaler', StandardScaler())
 ])
 cat_pipeline = Pipeline([ 
('simple_imp', SimpleImputer(strategy='most_frequent')) , 
('one_hot_enc', OneHotEncoder(handle_unknown='ignore'))
])
 comp_pipeline = ColumnTransformer([('num_pipeline', num_pipeline, numerical_features), 
('cat_pipeline', cat_pipeline, categorical_features)
 ])
 X_train_prepared = comp_pipeline.fit_transform(X_train)
 X_test_prepared = comp_pipeline.transform(X_test )
classifier_rf = RandomForestClassifier(max_depth = 2,random_state = 0 )<compute_test_metric>
results = []
model_evaluation(classifier_rf )<import_modules>
lr_clf = LinearRegression()
%time lr_clf.fit(X_train_prepared, y_train)
 y_train_predicted = lr_clf.predict(X_train_prepared)
 results.append({'alg': 'lr_clf', 'model': lr_clf, 'mse': mean_squared_error(y_train, y_train_predicted), 'mae': mean_absolute_error(y_train, y_train_predicted)})
 result=results[-1]
print('Algorithm:', result['alg'], 'MSE:', result['mse'], 'MAE:', result['mae'] )
from sklearn.neighbors import KNeighborsClassifier<import_modules>
rf_clf = RandomForestRegressor(random_state=RANDOM_SEED)
 %time rf_clf.fit(X_train_prepared, y_train)
 y_train_predicted = rf_clf.predict(X_train_prepared)
 results.append({'alg': 'rf_clf', 'model': rf_clf, 'mse': mean_squared_error(y_train, y_train_predicted), 'mae': mean_absolute_error(y_train, y_train_predicted)})
 result=results[-1]
print('Algorithm:', result['alg'], 'MSE:', result['mse'], 'MAE:', result['mae'] )
from sklearn.neighbors import KNeighborsClassifier<import_modules>
ada_clf=AdaBoostRegressor(random_state=RANDOM_SEED, n_estimators=100)
 %time ada_clf.fit(X_train_prepared,y_train)
 y_train_predicted = ada_clf.predict(X_train_prepared)
 print('Feature Importance Values
',ada_clf.feature_importances_)
 results.append({'alg': 'ada_clf', 'model': ada_clf, 'mse': mean_squared_error(y_train, y_train_predicted), 'mae': mean_absolute_error(y_train, y_train_predicted)})
 result=results[-1]
print('Algorithm:', result['alg'], 'MSE:', result['mse'], 'MAE:', result['mae'] )
from sklearn.neighbors import KNeighborsClassifier<choose_model_class>
xgb_clf = XGBRegressor(objective="reg:squarederror", seed=RANDOM_SEED)
 %time xgb_clf.fit(X_train_prepared, y_train)
 y_train_predicted = xgb_clf.predict(X_train_prepared)
 results.append({'alg': 'xgb_clf', 'model': xgb_clf, 'mse': mean_squared_error(y_train, y_train_predicted), 'mae': mean_absolute_error(y_train, y_train_predicted)})
 result=results[-1]
print('Algorithm:', result['alg'], 'MSE:', result['mse'], 'MAE:', result['mae'] )
classifier_knn = KNeighborsClassifier(leaf_size = 7, n_neighbors = 3,p = 1 )<find_best_params>
lso_clf = Lasso(random_state=RANDOM_SEED)
 %time lso_clf.fit(X_train_prepared, y_train)
 y_train_predicted = lso_clf.predict(X_train_prepared)
 results.append({'alg': 'lso_clf', 'model': lso_clf, 'mse': mean_squared_error(y_train, y_train_predicted), 'mae': mean_absolute_error(y_train, y_train_predicted)})
 result=results[-1]
print('Algorithm:', result['alg'], 'MSE:', result['mse'], 'MAE:', result['mae'] )
leaf_size = list(range(1,50))
n_neighbors = list(range(1,30))
p=[1,2]
hyperparameters = dict(leaf_size=leaf_size, n_neighbors=n_neighbors, p=p)
 grid_search_cv(classifier_knn,hyperparameters )<train_model>
eln_clf = ElasticNet(random_state=RANDOM_SEED)
 %time eln_clf.fit(X_train_prepared, y_train)
 y_train_predicted = eln_clf.predict(X_train_prepared)
 results.append({'alg': 'eln_clf', 'model': eln_clf, 'mse': mean_squared_error(y_train, y_train_predicted), 'mae': mean_absolute_error(y_train, y_train_predicted)})
 result=results[-1]
print('Algorithm:', result['alg'], 'MSE:', result['mse'], 'MAE:', result['mae'] )
model(classifier_knn )<compute_test_metric>
lgbm_clf = LGBMRegressor(random_state=RANDOM_SEED)
 %time lgbm_clf.fit(X_train_prepared, y_train)
 y_train_predicted = lgbm_clf.predict(X_train_prepared)
 results.append({'alg': 'lgbm_clf', 'model': lgbm_clf, 'mse': mean_squared_error(y_train, y_train_predicted), 'mae': mean_absolute_error(y_train, y_train_predicted)})
 result=results[-1]
print('Algorithm:', result['alg'], 'MSE:', result['mse'], 'MAE:', result['mae'] )
model_evaluation(classifier_knn )<define_variables>
print('
-------------------------Results-----------------------')
 for result in sorted(results, key=lambda k: k['mae']):
print('Algorithm:', result['alg'], 'MSE:', result['mse'], 'MAE:', result['mae'] )
models = {'MODELS':['LOGISTIC REGRESSION','SUPPORT VECTOR CLASSIFIER','DECISION TREE CLASSIFIER','RANDOM FOREST CLASSIFIER','K-NEAREST NEIGHBORS'],
'CROSS VAL ACCURACY(%)':[78.73,79.19,81.59,78.29,81.10]}<prepare_output>
xgb_param_grid = {
'min_child_weight': [1],
'gamma': [0],
'subsample': [1.0],
'colsample_bytree': [0.2],
'max_depth': [3],
'n_estimators': [2500]
}
xgb_grid_search = GridSearchCV(xgb_clf, xgb_param_grid, cv=CV_FOLDS, n_jobs=4, return_train_score=True, verbose=1)
 %time xgb_grid_search.fit(X_train_prepared, y_train)
 print(xgb_grid_search.best_params_)
 print(xgb_grid_search.best_score_)
 y_train_predicted = xgb_grid_search.best_estimator_.predict(X_train_prepared)
 results.append({'alg': 'xgb_clf_tuned', 'model': xgb_grid_search.best_estimator_, 'mse': mean_squared_error(y_train, y_train_predicted), 'mae': mean_absolute_error(y_train, y_train_predicted), 'best_params': xgb_grid_search.best_params_} )
cross_val = pd.DataFrame(models)
 cross_val.head()<load_from_csv>
rf_param_grid = [{'ccp_alpha': [0.0],
'criterion': ['mse'],
'max_depth': [None],
'max_features': [None],
'max_leaf_nodes': [None],
'min_impurity_decrease': [0.0],
'oob_score': [True],
'min_impurity_split': [None],
'min_samples_leaf': [2],
'min_samples_split': [4],
'min_weight_fraction_leaf': [0.0],
'random_state': [42],
}
]
rf_grid_search = GridSearchCV(rf_clf, rf_param_grid, n_jobs=4, cv=CV_FOLDS, return_train_score=True, verbose=1)
 %time rf_grid_search.fit(X_train_prepared, y_train)
 print(rf_grid_search.best_params_)
 print(rf_grid_search.best_score_)
 y_train_predicted = rf_grid_search.best_estimator_.predict(X_train_prepared)
 results.append({'alg': 'rf_clf_tuned', 'model': rf_grid_search.best_estimator_, 'mse': mean_squared_error(y_train, y_train_predicted), 'mae': mean_absolute_error(y_train, y_train_predicted), 'best_params': rf_grid_search.best_params_} )
test = pd.read_csv('.. /input/titanic/test.csv')
 test.head()<drop_column>
ada_param_grid = [{'n_estimators': [2000],
'loss': ['square'],
'random_state': [RANDOM_SEED]
}
]
ada_grid_search = GridSearchCV(ada_clf, ada_param_grid, n_jobs=4, cv=CV_FOLDS, return_train_score=True, verbose=1)
 %time ada_grid_search.fit(X_train_prepared, y_train)
 print(ada_grid_search.best_params_)
 print(ada_grid_search.best_score_)
 y_train_predicted = ada_grid_search.best_estimator_.predict(X_train_prepared)
 results.append({'alg': 'ada_clf_tuned', 'model': ada_grid_search.best_estimator_, 'mse': mean_squared_error(y_train, y_train_predicted), 'mae': mean_absolute_error(y_train, y_train_predicted)})
 result=results[-1]
print('Algorithm:', result['alg'], 'MSE:', result['mse'], 'MAE:', result['mae'] )
passenger_id = test['PassengerId']
test = test.drop(columns = ['PassengerId','Age','SibSp','Parch','Ticket','Cabin','Embarked'])
 test.head()<count_missing_values>
<count_missing_values><EOS> 
print('
-------------------------Results-----------------------')
 for result in sorted(results, key=lambda k: k['mae']):
print('Algorithm:', result['alg'], 'MSE:', result['mse'], 'MAE:', result['mae'])
 best_model = sorted(results, key=lambda k: k['mae'])[0]['model']
print(best_model)
 y_test_predicted = np.exp(best_model.predict(X_test_prepared))
output = pd.DataFrame({'Id': X_test.index, 'SalePrice': y_test_predicted})
 output.to_csv('/kaggle/working/submission.csv', index=False )