Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<drop_column>
import pandas as pd
import numpy as np
from sklearn.preprocessing import OrdinalEncoder, StandardScaler
from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_squared_error
from hyperopt import fmin, tpe, hp, anneal, Trials
import lightgbm
import xgboost
import warnings
train_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],
inplace=True,axis=1)
 test_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],
inplace=True,axis=1 )<drop_column>
houses_train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv', index_col='Id')
 houses_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv', index_col='Id' )
train_data.drop('PurchDate',axis=1,inplace=True)
 test_data.drop("PurchDate",axis=1,inplace=True )<drop_column>
houses_train.duplicated().sum()
train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!='object'<drop_column>
categorical_columns, quantitive_columns, int_columns = [], [], []
for i in houses_train.columns:
if houses_train[i].dtype == 'O':
categorical_columns.append(i)
 elif houses_train[i].dtype == 'float64':
quantitive_columns.append(i)
 else:
int_columns.append(i)
 int_columns.remove('SalePrice')
 print(len(categorical_columns)+ len(int_columns)+ len(quantitive_columns))
not_categorical=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!='object']<feature_engineering>
prep = pd.concat([houses_train, houses_test])
 prep.info()
for i in not_categorical:
maximum=np.max(train_data[i])
 train_data[i]=train_data[i]/maximum
maximum_test=np.max(test_data[i])
 test_data[i]=test_data[i]/maximum_test<drop_column>
print('Missing values Train:', houses_train.isna().sum().sum())
 print('Missing values Test:', houses_test.isna().sum().sum() )
categorical=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes=='object']<filter>
for i in ['Alley', 'Fence']:
for j in [houses_train, houses_test]:
j[i] = j[i].fillna(0)
 j[i] = j[i].apply(lambda x: x if x == 0 else 1 )
train_data[categorical[0]]<categorify>
for i in ['MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath',
'GarageYrBlt', 'GarageCars', 'GarageArea']:
for j in [houses_train, houses_test]:
j[i] = j[i].fillna(0 )
pd.get_dummies(train_data[categorical[0]] )<categorify>
for i in ['LotFrontage']:
average = houses_train[i].mean()
for j in [houses_train, houses_test]:
j[i] = j[i].fillna(average )
for i in categorical:
dummies=pd.get_dummies(train_data[i])
 dummies.columns=str(i)+'_'+dummies.columns
train_data=pd.concat([train_data,dummies],axis=1)
 train_data.drop(i,inplace=True,axis=1)
 dummies=pd.get_dummies(test_data[i])
 dummies.columns=str(i)+'_'+dummies.columns
test_data=pd.concat([test_data,dummies],axis=1)
 test_data.drop(i,inplace=True,axis=1 )<feature_engineering>
for i in ['MSZoning', 'Exterior1st', 'Exterior2nd', 'Electrical', 'KitchenQual', 'SaleType']:
mode = houses_train[i].mode() [0]
for j in [houses_train, houses_test]:
j[i] = j[i].fillna(mode )
for i in train_data.drop('IsBadBuy',axis=1 ).columns:
if i not in test_data.columns:
test_data[i]=np.zeros(len(test_data))<feature_engineering>
for i in ['Utilities', 'MasVnrType', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'Functional',
'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'PoolQC', 'MiscFeature']:
for j in [houses_train, houses_test]:
j[i] = j[i].fillna('None' )
for i in test_data.columns:
if i not in train_data.columns:
train_data[i]=np.zeros(len(train_data))<drop_column>
print('Missing values Train:', houses_train.isna().sum().sum())
 print('Missing values Test:', houses_test.isna().sum().sum() )
test_data=test_data[train_data.drop("IsBadBuy",axis=1 ).columns]<prepare_x_and_y>
rank_columns = []
warnings.filterwarnings(action='ignore')
 for i in prep.columns:
count = prep[(prep[i] == 'Ex')|(prep[i] == 'TA')|(prep[i] == 'Fa')|(prep[i] == 'Po')]['KitchenQual'].count()
if count > 0:
rank_columns.append(i)
 warnings.filterwarnings(action='default')
 print(rank_columns )
X=train_data.drop(['RefId','IsBadBuy'],axis=1)
 y=train_data['IsBadBuy']<split>
houses_train[rank_columns].apply(pd.Series.value_counts )
X_train,X_test,y_train,y_test=train_test_split(X,y,random_state=42 )<import_modules>
for i in rank_columns:
for j in [houses_train, houses_test]:
j[i]= j[i].map({'None' : 0, 'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5} )
from sklearn.neighbors import KNeighborsClassifier<import_modules>
outliers = pd.DataFrame({'column' : [], 'min' : [], 'max' : [], 'mean' : [], 'iqr' : [], 'lower_threshold' : [],
'upper_threshold' : [], 'outliers_qty' : []})
 for i in quantitive_columns:
df = houses_train[houses_train[i] != 0][i]
q1 = df.quantile(0.25)
 q3 = df.quantile(0.75)
 cl_min, cl_max, cl_mean = df.min() , df.max() , df.mean()
iqr = q3 - q1
threshold_min = q1 - 3*iqr
threshold_max = q3 + 3*iqr
qty = df[df < threshold_min].count() + df[df > threshold_max].count()
outliers = outliers.append({'column' : i, 'min' : cl_min, 'max' : cl_max, 'mean' : cl_mean, 'iqr' : iqr,
'lower_threshold' : threshold_min, 'upper_threshold' : threshold_max, 'outliers_qty' : qty},
ignore_index=True)
 outliers.sort_values(by='outliers_qty', ascending=False )
from sklearn.neighbors import KNeighborsClassifier<import_modules>
outliers.set_index('column', drop=True, inplace=True)
 outliers = outliers.to_dict(orient='index' )
from sklearn.neighbors import KNeighborsClassifier<train_model>
outliers_index_list = []
for i in ['LotArea', 'LotFrontage', 'OpenPorchSF', 'WoodDeckSF', 'TotalBsmtSF', 'GrLivArea']:
a = list(houses_train[(houses_train[i] < outliers[i]['lower_threshold'])| 
(houses_train[i] > outliers[i]['upper_threshold'])].index)
 for i in a:
outliers_index_list.append(i)
 outliers_index_list = list(set(outliers_index_list))
print(len(outliers_index_list))
KNN=KNeighborsClassifier(n_neighbors=11)
 KNN.fit(X_train,y_train )<compute_test_metric>
scaler = StandardScaler()
warnings.filterwarnings('ignore')
 houses_train.loc[:, quantitive_columns] = scaler.fit_transform(houses_train.loc[:, quantitive_columns])
 houses_test.loc[:, quantitive_columns] = scaler.transform(houses_test.loc[:, quantitive_columns])
 warnings.filterwarnings('default' )
KNN.score(X_test,y_test )<predict_on_test>
houses_train.shape[0]
houses_full = pd.concat([houses_train, houses_test] )
predict=KNN.predict(test_data.drop('RefId',axis=1))<prepare_output>
houses_full = pd.get_dummies(houses_full, drop_first=True)
 houses_train_ohe = houses_full[:houses_train.shape[0]]
houses_test_ohe = houses_full[houses_train.shape[0]:].drop('SalePrice', axis=1 )
Submission=pd.DataFrame(data=predict,columns=['IsBadBuy'])
 Submission.head()<prepare_output>
houses_train_ord = houses_train.copy()
houses_test_ord = houses_test.copy()
Submission['RefId']=test_data['RefId']
Submission.set_index('RefId',inplace=True )<save_to_csv>
encoder = OrdinalEncoder()
encoder.fit(pd.concat([houses_train[categorical_columns], houses_test[categorical_columns]]))
houses_train_ord[categorical_columns] = encoder.transform(houses_train[categorical_columns] ).astype('int64')
 houses_test_ord[categorical_columns] = encoder.transform(houses_test[categorical_columns] ).astype('int64' )
Submission.head()
Submission.to_csv('Submission.csv' )<load_from_csv>
x_houses_train_ord = houses_train_ord.drop('SalePrice', axis=1)
 y_houses_train_ord = houses_train_ord['SalePrice']
x_houses_train_ohe = houses_train_ohe.drop('SalePrice', axis=1)
 y_houses_train_ohe = houses_train_ohe['SalePrice']
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 subm = pd.read_csv('.. /input/sample_submission.csv')
<groupby>
lgb_space = {'n_estimators' : hp.quniform('n_estimators', 1000, 3000, 50),
'learning_rate': hp.uniform('learning_rate', 0.00001, 0.4),
'max_depth': hp.quniform('max_depth', 3, 30, 1),
'num_leaves' : hp.quniform('num_leaves', 5, 31, 5),
'min_child_samples' : hp.quniform('min_child_samples', 5, 41, 5),
'subsample' : hp.uniform('subsample', 0.6, 1),
'colsample_bytree' : hp.uniform('colsample_bytree', 0.6, 1),
'reg_alpha' : hp.uniform('reg_alpha', 1, 15),
'reg_lambda' : hp.uniform('reg_lambda', 1, 15)}
train.groupby('author' ).size() 
<import_modules>
def lgb_hyperopt(params):
params = {'n_estimators': int(params['n_estimators']),
'learning_rate': params['learning_rate'],
'max_depth': int(params['max_depth']),
'num_leaves' : int(params['num_leaves']),
'min_child_samples' : int(params['min_child_samples']),
'subsample' : params['subsample'],
'colsample_bytree' : params['colsample_bytree'],
'reg_alpha' : params['reg_alpha'],
'reg_lambda' : params['reg_lambda']
}
lgb = lightgbm.LGBMRegressor(**params)
 score = cross_val_score(lgb, x_houses_train_ord, y_houses_train_ord, scoring='neg_mean_squared_error',
cv=5, n_jobs=-1 ).mean()
return -score
from sklearn import preprocessing<categorify>
lgb_best = fmin(fn=lgb_hyperopt, space=lgb_space, max_evals=40, rstate=np.random.RandomState(12345), algo=tpe.suggest )
le = preprocessing.LabelEncoder()
le.fit(train.author)
 num_of_labels = len(list(le.classes_))
list(le.classes_)
<categorify>
lgb_final = lightgbm.LGBMRegressor(n_estimators = int(lgb_best['n_estimators']), learning_rate = lgb_best['learning_rate'],
max_depth = int(lgb_best['max_depth']), num_leaves = int(lgb_best['num_leaves']),
min_child_samples = int(lgb_best['min_child_samples']), subsample = lgb_best['subsample'],
colsample_bytree = lgb_best['colsample_bytree'], reg_alpha = lgb_best['reg_alpha'],
reg_lambda = lgb_best['reg_lambda'] )
y_labels = le.transform(train.author)
 y_labels<categorify>
lgb_final.fit(x_houses_train_ord, y_houses_train_ord, categorical_feature=categorical_columns )
le.inverse_transform(y_labels )<create_dataframe>
mean_squared_error(y_houses_train_ord, lgb_final.predict(x_houses_train_ord), squared=False )
row, column = np.where(pd.isnull(train))
print(row, column )<string_transform>
submission_lgb = pd.DataFrame({'Id': houses_test.index, 'SalePrice' : lgb_final.predict(houses_test_ord)} )
re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
 def tokenize(s): return re_tok.sub(r' \1 ', s ).split()<define_variables>
submission_lgb.to_csv('new_submission_lgb.csv', index=False )
n = train.shape[0]<categorify>
xgb_space = {'n_estimators':hp.quniform('n_estimators', 1000, 4000, 50),
'gamma':hp.uniform('gamma', 0.01, 0.1),
'learning_rate':hp.uniform('learning_rate', 0.00001, 0.4),
'max_depth':hp.quniform('max_depth', 3,30,1),
'subsample':hp.uniform('subsample', 0.60, 0.95),
'colsample_bytree':hp.uniform('colsample_bytree', 0.60, 0.98),
'colsample_bylevel':hp.uniform('colsample_bylevel', 0.60, 0.98),
'reg_lambda': hp.uniform('reg_lambda', 1, 20)
 }
TEXT = 'text'
vec = TfidfVectorizer(ngram_range=(1,2), tokenizer=tokenize,
min_df=3, max_df=0.9, strip_accents='unicode', use_idf=1,
smooth_idf=1, sublinear_tf=1)
 trn_term_doc = vec.fit_transform(train[TEXT])
 test_term_doc = vec.transform(test[TEXT] )<compute_test_metric>
def xgb_hyperopt(params):
params = {'n_estimators': int(params['n_estimators']),
'gamma': params['gamma'],
'learning_rate': params['learning_rate'],
'max_depth': int(params['max_depth']),
'subsample': params['subsample'],
'colsample_bytree': params['colsample_bytree'],
'colsample_bylevel': params['colsample_bylevel'],
'reg_lambda': params['reg_lambda']
}
xgb = xgboost.XGBRegressor(**params)
 score = cross_val_score(xgb, x_houses_train_ohe, y_houses_train_ohe, scoring='neg_mean_squared_error',
cv=5, n_jobs=-1 ).mean()
return -score
trn_term_doc, test_term_doc<prepare_x_and_y>
xgb_best = fmin(fn=xgb_hyperopt, space=xgb_space, max_evals=40, rstate=np.random.RandomState(12345), algo=tpe.suggest )
x = trn_term_doc
test_x = test_term_doc<train_model>
xgb_final = xgboost.XGBRegressor(n_estimators = int(xgb_best['n_estimators']), colsample_bytree = xgb_best['colsample_bytree'],
gamma = xgb_best['gamma'], learning_rate = xgb_best['learning_rate'],
max_depth = int(xgb_best['max_depth']), subsample = xgb_best['subsample'],
colsample_bylevel = xgb_best['colsample_bylevel'], reg_lambda = xgb_best['reg_lambda'] )
def get_model(y):
m = LogisticRegression(C=4, dual=True)
 return m.fit(x, y )<predict_on_test>
xgb_final.fit(x_houses_train_ohe, y_houses_train_ohe )
preds = np.zeros(( len(test), num_of_labels))
m = get_model(y_labels)
 preds = m.predict_proba(test_x )<save_to_csv>
mean_squared_error(y_houses_train_ohe, xgb_final.predict(x_houses_train_ohe), squared=False )
submid = pd.DataFrame({'id': subm["id"]})
 submission = pd.concat([submid, pd.DataFrame(preds, columns = ['EAP','HPL','MWS'])], axis=1)
 submission.to_csv('submission.csv', index=False)
<import_modules>
submission_xgb = pd.DataFrame({'Id': houses_test.index, 'SalePrice' : xgb_final.predict(houses_test_ohe)} )
<import_modules><EOS> 
submission_xgb.to_csv('new_submission_xgb.csv', index=False )