Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<data_type_conversions>
%matplotlib inline
if not sys.warnoptions:
warnings.simplefilter("ignore" )
datecols=traindata.select_dtypes(include="datetime" ).columns.tolist()
traindata[datecols]=traindata[datecols].astype("int64")
 testdf[datecols]=testdf[datecols].astype("int64" )<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
procs = [FillMissing, Categorify, Normalize]
dep_var = 'Sales'
cont_names=[
'CompetitionDistance',
'Week',
'Day',
'Dayofyear',
'Elapsed',
'ratio-sales-customer',
'ratio-saturday-week',
'ratio-sunday-week',
'ratio-promo-nopromo',
'Promo_thisweek',
'Open_thisweek',
'StateHolidayBool_thisweek',
'SchoolHoliday_thisweek',
'Promo_prevweek',
'Open_prevweek',
'StateHolidayBool_prevweek',
'SchoolHoliday_prevweek',
'Promo_nextweek',
'Open_nextweek',
'StateHolidayBool_nextweek',
'SchoolHoliday_nextweek',
'Promo2Days',
'CompetitionDaysOpen',
'trend',
'trend_DE',
'Max_Humidity',
'Max_Wind_SpeedKm_h',
'Mean_Humidity',
'Mean_TemperatureC',
'Max_TemperatureC_chnage',
'Month_Sales_mean',
'Year_Sales_mean',
'Dayofweek_Sales_mean',
'Dayofweek_promo_Sales_mean',
'BeforeSchoolHoliday',
'AfterSchoolHoliday',
'BeforeClosed',
'AfterClosed',
'BeforePromo',
'AfterPromo',
'BeforeStateHolidayBool',
'AfterStateHolidayBool',
'Promo2ActiveMonthBool',
'BeforePromo2ActiveMonthBool',
'AfterPromo2ActiveMonthBool',
'SchoolHoliday_fw',
'StateHolidayBool_fw',
'Promo_fw',
'Closed_fw',
'Promo2ActiveMonthBool_fw',
'CompetitionOpenSince', 'Promo2Since'
]
cat_names=[
'Store',
'DayOfWeek',
'Open',
'Promo',
'StateHoliday',
'SchoolHoliday',
'StoreType',
'Assortment',
'Promo2',
'PromoInterval',
'Year',
'Month',
'Dayofweek',
'Is_month_end',
'Is_month_start',
'Is_quarter_end',
'Is_quarter_start',
'Is_year_end',
'Is_year_start',
'Promo2SinceYear',
'Promo2Na',
'Events',
'Fog',
'Hail',
'Rain',
'Snow',
'Thunderstorm',
'Quarter',
'CompetitionOpenNA',
'CompetitionDistanceNA',
'CompetitionOpenSinceYear',
'State'
] 
<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
max_log_y = np.log(np.max(traindata['Sales']))
y_range = torch.tensor([0, max_log_y], device=defaults.device )<categorify>
train = train.drop(train[(train['GrLivArea']>4500)&(train['SalePrice']<300000)].index )
databunch =(TabularList.from_df(traindata, path="", cat_names=cat_names, cont_names=cont_names, procs=procs,)
.split_by_idx(valid_idx)
.label_from_df(cols=dep_var, label_cls=FloatList, log=True)
.add_test(TabularList.from_df(testdf, path=path, cat_names=cat_names, cont_names=cont_names)) 
.databunch())
 databunch.batch_size=s.batchsize<choose_model_class>
train['SalePrice'] = np.log1p(train['SalePrice'] )
learn = tabular_learner(databunch, layers=[s.l1,s.l2], ps=[s.ps1,s.ps2], emb_drop=s.emb_drop, y_range=y_range, metrics=exp_rmspe )<find_best_params>
train_corr = train
train_features = train.drop('SalePrice', axis = 1)
 y = train.SalePrice.reset_index(drop=True)
 test_features = test
features = pd.concat([train_features, test_features] )
learn.model<find_best_params>
def missing_vals(df):
total = df.isnull().sum().sort_values(ascending = False)
 percent =(total/len(df)*100)
 missing = percent[percent>0]
return pd.DataFrame(missing, columns = ['Percent'])
 missing_vals(features )
learn.lr_find()<train_model>
features = features.dropna(thresh = len(features)*0.75, axis=1)
 for i in 'MSSubClass YrSold MoSold GarageYrBlt'.split() :
features[i] = features[i].apply(lambda x: str(x))
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
for i in 'Utilities Functional Electrical KitchenQual MSZoning'.split() :
features[i] = features[i].fillna(features[i].mode() [0])
 for i in 'GarageType GarageFinish GarageQual GarageCond'.split() :
features[i] = features[i].fillna('None')
 for i in 'GarageYrBlt GarageArea GarageCars'.split() :
features[i] = features[i].fillna(0)
 for i in 'BsmtFinSF1 BsmtFinSF2 BsmtUnfSF TotalBsmtSF BsmtFullBath BsmtHalfBath'.split() :
features[i] = features[i].fillna(0)
 for i in 'BsmtQual BsmtCond BsmtExposure BsmtFinType1 BsmtFinType2'.split() :
features[i] = features[i].fillna('None')
 features["MasVnrType"] = features["MasVnrType"].fillna("None")
 features["MasVnrArea"] = features["MasVnrArea"].fillna(0)
 for i in 'Exterior1st Exterior2nd SaleType'.split() :
features[i] = features[i].fillna(features[i].mode() [0])
 features['MSSubClass'] = features['MSSubClass'].fillna("None" )
learn.fit_one_cycle(s.l1epoch, s.lrate, wd=s.wd )<train_model>
missing_vals(features )
learn.fit_one_cycle(s.l2epoch, s.lrate/s.lrate_ratio, wd=s.wd )<train_model>
features['TotSqrSF'] = features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
learn.fit_one_cycle(s.l3epoch, s.lrate/(s.lrate_ratio*s.lrate_ratio), wd=s.wd )<predict_on_test>
feature_en = 'PoolArea Fireplaces GarageArea TotalBsmtSF 2ndFlrSF'.split()
for i in feature_en:
features['Has'+i] = features[i].apply(lambda x: 1 if x> 0 else 0 )
valid_preds=learn.get_preds(DatasetType.Valid)
 traindata["SalesPreds"]=pd.Series(index=traindata.iloc[valid_idx].index,data=np.exp(valid_preds[0].numpy().T[0]))<compute_test_metric>
skewed_features = skewness.index
for feat in skewed_features:
features[feat] = boxcox1p(features[feat],0.5 )
def rmspe_metric(act,pred):
return np.sqrt(np.mean(((act-pred)/act)**2))<compute_train_metric>
features_cat = features.copy()
rmspe_metric(traindata.Sales,traindata.SalesPreds )<compute_train_metric>
features = pd.get_dummies(features, drop_first = True)
 features.shape
store_rmspe=traindata.groupby(["Store"] ).apply(lambda x:rmspe_metric(x.Sales,x.SalesPreds)).sort_values(ascending=False )<rename_columns>
train_set=features[:len(y)]
test_set=features[len(train_set):]
train_set.head()
t=traindata.set_index("Date" )<save_to_csv>
train_set.drop('Id', axis=1, inplace=True, errors='raise')
 test_set.drop('Id', axis=1, inplace=True, errors='raise' )
test_preds=learn.get_preds(DatasetType.Test)
 testdf["Sales"]=np.exp(test_preds[0].data ).numpy().T[0]
testdf[["Id","Sales"]]=testdf[["Id","Sales"]].astype("int")
 testdf[["Id","Sales"]].to_csv("rossmann_submission.csv",index=False )<set_options>
X = train_set
test_X = test_set
%matplotlib inline<load_from_csv>
X_train = X
X_test = test_X
y_train = y
def rmse(model):
rmse = np.sqrt(-cross_val_score(model, X_train, y_train, scoring="neg_mean_squared_error", cv = 10))
return(rmse )
train_scores = pd.read_csv('/kaggle/input/trends-assessment-prediction/train_scores.csv' ).sort_values(by='Id')
 loadings = pd.read_csv('/kaggle/input/trends-assessment-prediction/loading.csv')
 fnc = pd.read_csv('/kaggle/input/trends-assessment-prediction/fnc.csv')
 sample = pd.read_csv('/kaggle/input/trends-assessment-prediction/sample_submission.csv')
 reveal = pd.read_csv('/kaggle/input/trends-assessment-prediction/reveal_ID_site2.csv')
 icn_nums = pd.read_csv('/kaggle/input/trends-assessment-prediction/ICN_numbers.csv')
<import_modules>
model_random_forest = make_pipeline(RobustScaler() , RandomForestRegressor(n_estimators = 100 , oob_score = True, n_jobs = -1))
model_random_forest.fit(X_train, y_train)
 randomforest_rmse = rmse(model_random_forest ).mean()
import re
from tqdm import tqdm<feature_engineering>
ridge = make_pipeline(RobustScaler() , linear_model.Ridge())
 ridge.fit(X_train, y_train)
 ridge_rmse = rmse(ridge ).mean()
r = re.compile('\d+')
 col_dict = {}
for col in fnc.columns:
ind = r.findall(col)
 if ind:
col_dict[col] = [int(i)for i in ind]<prepare_x_and_y>
xg_boost = make_pipeline(RobustScaler() , xgb.XGBRegressor())
 xg_boost.fit(X_train, y_train)
 xgboost_rmse = rmse(xg_boost ).mean()
def get_matrix(df_row, return_idx=False):
matrix = np.zeros(( 100, 100))
for col in df_row.index[1:]:
i, j = col_dict[col]
matrix[i, j] = df_row[col]
matrix += matrix.T
idx = np.array([ 2, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 16, 17, 18, 20, 21, 23,
27, 32, 33, 37, 38, 40, 43, 45, 48, 51, 53, 54, 55, 56, 61, 62, 63,
66, 67, 68, 69, 70, 71, 72, 77, 79, 80, 81, 83, 84, 88, 93, 94, 96,
98, 99])
 if return_idx:
return matrix[:, idx][idx, :], idx
return matrix[:, idx][idx, :]<create_dataframe>
results_df = pd.DataFrame({'Model': ['Lasso', 'RandomForest', 'Ridge', 'XGBoost'], 'Score': [lasso_rmse, randomforest_rmse,ridge_rmse, xgboost_rmse]})
 results_df = results_df.sort_values(by='Score', ascending=True ).reset_index(drop=True)
 results_df
_, idx = get_matrix(fnc.iloc[0], return_idx=True)
 degrees = pd.DataFrame(degrees, columns=idx)
 degrees['Id'] = fnc['Id']<define_variables>
df = pd.DataFrame({'Id': test.Id, 'SalePrice':np.exp(model_lasso.predict(X_test)) })
 df.to_csv('Submission_final.csv', index = False )
len(glob('/kaggle/input/trends-assessment-prediction/fMRI_train/*.mat')) , len(glob('/kaggle/input/trends-assessment-prediction/fMRI_test/*.mat'))<import_modules>
cat_high_corr = ['HeatingQC','GarageType','MSZoning', 'Neighborhood', 'Condition2', 'MasVnrType', 'ExterQual', 'BsmtQual','CentralAir', 'KitchenQual', 'SaleType', 'SaleType']
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import Ridge, Lasso
from sklearn.svm import SVR
from sklearn.model_selection import GridSearchCV, KFold, cross_val_predict<merge>
num_features = train_corr.select_dtypes(include = np.number )
train_ids = sorted(loadings[loadings['Id'].isin(train_scores.Id)]['Id'].values)
 test_ids = sorted(loadings[~loadings['Id'].isin(train_scores.Id)]['Id'].values)
 test_prediction = pd.DataFrame(test_ids, columns=['Id'], dtype=str)
 target_columns =('age', 'domain1_var1', 'domain1_var2','domain2_var1','domain2_var2')
 fnc_columns = fnc.columns[1:]
degrees_columns = degrees.columns[:-1]
data = pd.merge(loadings, train_scores, on='Id')
 data = pd.merge(data, fnc, on='Id')
 data = pd.merge(data, degrees, on='Id')
 X_test = pd.merge(loadings[loadings.Id.isin(test_ids)], fnc, on='Id')
 X_test = pd.merge(X_test, degrees, on='Id' ).drop('Id', axis=1 )<compute_test_metric>
num_corr = num_features.corr() ['SalePrice'][:-1]
golden_features_list = num_corr[abs(num_corr)> 0.3].sort_values(ascending=False)
 golden_features_list
def MAPE(y_true, y_pred, **kwargs):

 return np.sum(np.abs(y_true - y_pred)) / y_true.sum()
mape_scorer = make_scorer(MAPE, greater_is_better=False )<find_best_model_class>
features = features_cat[cat_high_corr + list(golden_features_list.index)]
best_models = {}
total_score = []
for col in target_columns:
X_train = data.dropna(subset=[col], axis=0 ).drop(list(target_columns), axis=1 ).drop('Id', axis=1)
 X_train[fnc_columns] /= 500
y_train = data.dropna(subset=[col], axis=0)[col]
gs.fit(X_train, y_train)
 best_models[col] = gs.best_estimator_
y_pred = cross_val_predict(gs.best_estimator_, X_train, y_train, cv=cv, n_jobs=-1)
 total_score.append(MAPE(y_train, y_pred))
print(col, MAPE(y_train, y_pred))
total_score = np.array(total_score)
 print(f'Total score: {np.sum(total_score*[.3,.175,.175,.175,.175])}' )<predict_on_test>
features_cat = pd.get_dummies(features, drop_first = True )
def get_pred(col, model):
X_train = data.dropna(subset=[col], axis=0 ).drop(list(target_columns), axis=1 ).drop('Id', axis=1)
 X_train[fnc_columns] /= 500
y_train = data.dropna(subset=[col], axis=0)[col]
y_pred = cross_val_predict(model, X_train, y_train, cv=cv, n_jobs=-1)
 return y_pred<predict_on_test>
train_set=features_cat[:len(y)]
test_set=features_cat[len(train_set):]
train_set.head()
X_test[fnc_columns] /= 500
for col in target_columns:
test_prediction[col] = best_models[col].predict(X_test )<compute_test_metric>
X = train_set
test_X = test_set
<categorify>
X_train = X
X_test = test_X
y_train = y
def make_sub(test_prediction):

 target_columns =('age', 'domain1_var1', 'domain1_var2','domain2_var1','domain2_var2')
 _columns =(0,1,2,3,4)
 tst = test_prediction.rename(columns=dict(zip(target_columns, _columns)))
 tst = tst.melt(id_vars='Id',
value_vars=_columns,
value_name='Predicted')
 tst['target_type'] = tst.variable.map(dict(zip(_columns, target_columns)))
 tst['Id_'] = tst[['Id', 'target_type']].apply(lambda x: '_'.join(( str(x[0]), str(x[1]))), axis=1)
 return tst.sort_values(by=['Id', 'variable'])\ 
.drop(['Id', 'variable', 'target_type'],axis=1)\ 
.rename(columns={'Id_':'Id'})\ 
.reset_index(drop=True)\
[['Id', 'Predicted']]<prepare_output>
model_random_forest = make_pipeline(RobustScaler() , RandomForestRegressor(n_estimators = 200 , oob_score = True, n_jobs = -1))
model_random_forest.fit(X_train, y_train)
 randomforest_rmse = rmse(model_random_forest ).mean()
sub = make_sub(test_prediction)
 sub.head()<save_to_csv>
ridge = make_pipeline(RobustScaler() , linear_model.Ridge())
 ridge.fit(X_train, y_train)
 ridge_rmse = rmse(ridge ).mean()
sub.to_csv('ridge_mape_500.csv', index=False )<set_options>
xg_boost = make_pipeline(RobustScaler() , xgb.XGBRegressor())
 xg_boost.fit(X_train, y_train)
 xgboost_rmse = rmse(xg_boost ).mean()
%matplotlib inline<load_from_csv>
results_df = pd.DataFrame({'Model': ['Lasso', 'RandomForest', 'Ridge', 'XGBoost'], 'Score': [lasso_rmse, randomforest_rmse,ridge_rmse, xgboost_rmse]})
 results_df = results_df.sort_values(by='Score', ascending=True ).reset_index(drop=True)
 results_df