Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<data_type_conversions>
warnings.filterwarnings('ignore')
 %matplotlib inline
train_data["Nationality"].fillna(value="Nationality_unk",inplace=True)
 test_data["Nationality"].fillna(value="Nationality_unk",inplace=True )<count_values>
df_train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 ids = df_test['Id'].values
train_data["Size"].value_counts(dropna=False )<data_type_conversions>
y_train = df_train.SalePrice.values
x_train = df_train.drop('SalePrice', 1 )
train_data["Size"].fillna(value="Size_unk",inplace=True)
 test_data["Size"].fillna(value="Size_unk",inplace=True )<count_values>
total = df_train.isnull().sum().sort_values(ascending=False)
 percent =(df_train.isnull().sum() /df_train.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(20 )
train_data["TopThreeAmericanName"].value_counts(dropna=False )<data_type_conversions>
x_train = x_train.drop(( missing_data[missing_data['Total'] > 81] ).index,1)
 x_train = x_train.apply(lambda x:x.fillna(x.value_counts().index[0]))
x_train.isnull().sum().max()
train_data["TopThreeAmericanName"].fillna(value="TOP_unk",inplace=True)
 test_data["TopThreeAmericanName"].fillna(value="TOP_unk",inplace=True )<count_values>
df_test = df_test.drop(( missing_data[missing_data['Total'] > 81] ).index,1)
 df_test = df_test.apply(lambda x:x.fillna(x.value_counts().index[0]))
train_data["PRIMEUNIT"].value_counts(dropna=False )<data_type_conversions>
x_train.drop("Id", axis = 1, inplace = True)
 df_test.drop("Id", axis = 1, inplace = True )
train_data["PRIMEUNIT"].fillna(value="Prime_unk",inplace=True)
 test_data["PRIMEUNIT"].fillna(value="Prime_unk",inplace=True )<count_values>
cols = x_train.select_dtypes(include='object' ).columns
for c in cols:
lbl = LabelEncoder()
lbl.fit(list(x_train[c].values))
x_train[c] = lbl.transform(list(x_train[c].values))
df_test[c] = lbl.transform(list(df_test[c].values))
print('Shape all_data: {}'.format(x_train.shape))
train_data["AUCGUART"].value_counts(dropna=False )<count_values>
indexes = x_train[(df_train['GrLivArea']>4000)&(df_train['SalePrice']<300000)].index
x_train = x_train.drop(indexes)
 y_train = np.delete(y_train, indexes )
test_data["AUCGUART"].value_counts(dropna=False )<data_type_conversions>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
train_data["AUCGUART"].fillna(value="AU_unk",inplace=True)
 test_data["AUCGUART"].fillna(value="AU_unk",inplace=True )<drop_column>
model_xgb = xgb.XGBRegressor(n_estimators=2200 )
train_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],
inplace=True,axis=1)
 test_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],
inplace=True,axis=1 )<count_values>
n_folds = 5
def rmsle(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse = np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
train_data["PurchDate"].value_counts()<drop_column>
model_xgb.fit(x_train, y_train)
 xgb_train_pred = model_xgb.predict(x_train)
 xgb_pred = model_xgb.predict(df_test)
 print(rmsle(y_train, xgb_train_pred))
train_data.drop("PurchDate",axis=1,inplace=True)
 test_data.drop("PurchDate",axis=1,inplace=True )<drop_column>
sub = pd.DataFrame()
sub['Id'] = ids
sub['SalePrice'] = xgb_pred
sub.to_csv('submission.csv',index=False )
train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!='object'<drop_column>
model_xgb = xgb.XGBRegressor(reg_lambda=0.8571, n_estimators=2200, nthread = -1 )
not_categorical=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!='object']
not_categorical<feature_engineering>
model_xgb.fit(x_train, y_train)
 xgb_train_pred = model_xgb.predict(x_train)
 xgb_pred = model_xgb.predict(df_test)
 print(rmsle(y_train, xgb_train_pred))
for i in not_categorical:
maxium = max(train_data[i])
 train_data[i] = train_data[i]/maxium
maxium_test = max(test_data[i])
 test_data[i] = test_data[i]/maxium_test<drop_column>
params = {'min_child_weight':[4,5], 'gamma':[i/10.0 for i in range(3,6)], 'subsample':[i/10.0 for i in range(6,11)],
'colsample_bytree':[i/10.0 for i in range(4,11)], 'max_depth': [2,3,4], 'reg_lambda':[i/10.0 for i in range(7,9)], 'reg_alpha':[i/10.0 for i in range(4,7)]}
model = xgb.XGBRegressor(nthread=-1)
 grid = GridSearchCV(model, params)
 grid.fit(x_train, y_train )
categorical = train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes =='object']
categorical<filter>
xgb_train_pred = grid.best_estimator_.predict(x_train)
 xgb_pred = grid.best_estimator_.predict(df_test)
 print(rmsle(y_train, xgb_train_pred))
train_data[categorical[0]]<categorify>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
pd.get_dummies(train_data[categorical[0]] )<categorify>
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 GBoost = GradientBoostingRegressor(n_estimators=3000, random_state =42)
 ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005,random_state=42))
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=42))
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso )
for i in categorical:
dummies=pd.get_dummies(train_data[i])
 dummies.columns=str(i)+'_'+dummies.columns
train_data=pd.concat([train_data,dummies],axis=1)
 train_data.drop(i,axis=1,inplace=True)
 dummies=pd.get_dummies(test_data[i])
 dummies.columns=str(i)+'_'+dummies.columns
test_data=pd.concat([test_data,dummies],axis=1)
 test_data.drop(i,axis=1,inplace=True )<feature_engineering>
model_xgb = xgb.XGBRegressor(n_estimators=2200, nthread = -1)
 model_xgb.fit(x_train, y_train)
 model_lgb = lgb.LGBMRegressor(objective='regression',n_estimators=720)
 model_lgb.fit(x_train, y_train)
 stacked_averaged_models.fit(x_train.values, y_train )
for i in train_data.drop("IsBadBuy",axis=1 ).columns:
if i not in test_data.columns:
test_data[i]=np.zeros(len(test_data))<feature_engineering>
lgb_pred = model_lgb.predict(df_test)
 xgb_pred = model_xgb.predict(df_test)
 stacked_pred = stacked_averaged_models.predict(df_test.values )
for i in test_data:
if i not in train_data.columns:
train_data[i]=np.zeros(len(train_data))<prepare_x_and_y>
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15
X = train_data.drop(['RefId','IsBadBuy'],axis=1)
 y = train_data['IsBadBuy']<split>
sub = pd.DataFrame()
sub['Id'] = ids
sub['SalePrice'] = xgb_pred
sub.to_csv('submission.csv',index=False )
X_train,X_test,y_train,y_test = train_test_split(X,y )<train_model>
train = pd.read_csv('.. /input/ames-housing-dataset/AmesHousing.csv')
 train.drop(['PID'], axis=1, inplace=True)
 origin = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 train.columns = origin.columns
test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 print('Train:{} Test:{}'.format(train.shape,test.shape))
missing = test.isnull().sum()
missing = missing[missing>0]
train.drop(missing.index, axis=1, inplace=True)
 train.drop(['Electrical'], axis=1, inplace=True)
 test.dropna(axis=1, inplace=True)
 test.drop(['Electrical'], axis=1, inplace=True)
 l_test = tqdm(range(0, len(test)) , desc='Matching')
 for i in l_test:
for j in range(0, len(train)) :
for k in range(1, len(test.columns)) :
if test.iloc[i,k] == train.iloc[j,k]:
continue
else:
break
else:
submission.iloc[i, 1] = train.iloc[j, -1]
break
l_test.close()
submission.to_csv('result-with-best.csv', index=False )