Kaggle competition: House Prices - Advanced Regression Techniques
<choose_model_class><EOS> 
import pandas as pd
import numpy as np
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<merge>
import pandas as pd
import numpy as np
X_prediction['Patient'] = X_prediction['Patient_Week'].str.extract(r' (.*)_.*')
 X_prediction['Weeks'] = X_prediction['Patient_Week'].str.extract(r'.*_ (.*)' ).astype(int)
 X_prediction = X_prediction[['Patient', 'Weeks', 'Patient_Week']]
rename_cols = {'Weeks_y':'Min_week', 'Weeks_x': 'Weeks', 'FVC':'Base_FVC'}
X_prediction = X_prediction.merge(raw_test, how='left', left_on='Patient', right_on='Patient' ).rename(columns=rename_cols)[['Patient', 'Min_week', 'Base_FVC', 'Percent', 'Age', 'Sex', 'SmokingStatus', 'Weeks', 'Patient_Week']].reset_index(drop=True )<feature_engineering>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print("Train Shape: " + str(train.shape))
print("Test Shape: " + str(test.shape))
X_prediction['Base_week'] = X_prediction['Weeks'] - X_prediction['Min_week']<data_type_conversions>
unique_vals_per_col = train.T.apply(lambda x: x.nunique() , axis=1)
 print(unique_vals_per_col.head(5))
<categorify>
train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index)
class OneHotEncoder(SklearnOneHotEncoder):
def __init__(self, **kwargs):
super(OneHotEncoder, self ).__init__(**kwargs)
 self.fit_flag = False
def fit(self, X, **kwargs):
out = super().fit(X)
 self.fit_flag = True
return out
def transform(self, X, categories, index='', name='', **kwargs):
sparse_matrix = super(OneHotEncoder, self ).transform(X)
 new_columns = self.get_new_columns(X=X, name=name, categories=categories)
 d_out = pd.DataFrame(sparse_matrix.toarray() , columns=new_columns, index=index)
 return d_out
def fit_transform(self, X, categories, index, name, **kwargs):
self.fit(X)
 return self.transform(X, categories=categories, index=index, name=name)
 def get_new_columns(self, X, name, categories):
new_columns = []
for j in range(len(categories)) :
new_columns.append('{}_{}'.format(name, categories[j]))
return new_columns
def standardisation(x, u, s):
return(x-u)/s
def normalization(x, ma, mi):
return(x-mi)/(ma-mi)
 class data_preparation() :
def __init__(self, bool_normalization=True,bool_standard=False):
self.enc_sex = LabelEncoder()
self.enc_smok = LabelEncoder()
self.onehotenc_smok = OneHotEncoder()
self.standardisation = bool_standard
self.normalization = bool_normalization
def __call__(self, data_untransformed):
data = data_untransformed.copy(deep=True)
 try:
data['Sex'] = self.enc_sex.transform(data['Sex'].values)
 data['SmokingStatus'] = self.enc_smok.transform(data['SmokingStatus'].values)
 data = pd.concat([data.drop(columns=['SmokingStatus']), self.onehotenc_smok.transform(data['SmokingStatus'].values.reshape(-1,1), categories=self.enc_smok.classes_, name='', index=data.index ).astype(int)], axis=1)
 if self.standardisation:
data['Base_week'] = standardisation(data['Base_week'],self.base_week_mean,self.base_week_std)
 data['Base_FVC'] = standardisation(data['Base_FVC'],self.base_fvc_mean,self.base_fvc_std)
 data['Base_percent'] = standardisation(data['Base_percent'],self.base_percent_mean,self.base_percent_std)
 data['Age'] = standardisation(data['Age'],self.age_mean,self.age_std)
 data['Weeks'] = standardisation(data['Weeks'],self.weeks_mean,self.weeks_std)
 if self.normalization:
data['Base_week'] = normalization(data['Base_week'],self.base_week_max,self.base_week_min)
 data['Base_FVC'] = normalization(data['Base_FVC'],self.base_fvc_max,self.base_fvc_min)
 data['Percent'] = normalization(data['Percent'],self.base_percent_max,self.base_percent_min)
 data['Age'] = normalization(data['Age'],self.age_max,self.age_min)
 data['Weeks'] = normalization(data['Weeks'],self.weeks_max,self.weeks_min)
 data['Min_week'] = normalization(data['Min_week'],self.base_week_max,self.base_week_min)
 except NotFittedError:
data['Sex'] = self.enc_sex.fit_transform(data['Sex'].values)
 data['SmokingStatus'] = self.enc_smok.fit_transform(data['SmokingStatus'].values)
 data = pd.concat([data.drop(columns=['SmokingStatus']), self.onehotenc_smok.fit_transform(data['SmokingStatus'].values.reshape(-1,1), categories=self.enc_smok.classes_, name='', index=data.index ).astype(int)], axis=1)
 if self.standardisation:
self.base_week_mean = data['Base_week'].mean()
self.base_week_std = data['Base_week'].std()
data['Base_week'] = standardisation(data['Base_week'],self.base_week_mean,self.base_week_std)
 self.base_fvc_mean = data['Base_FVC'].mean()
self.base_fvc_std = data['Base_FVC'].std()
data['Base_FVC'] = standardisation(data['Base_FVC'],self.base_fvc_mean,self.base_fvc_std)
 self.base_percent_mean = data['Base_percent'].mean()
self.base_percent_std = data['Base_percent'].std()
data['Base_percent'] = standardisation(data['Base_percent'],self.base_percent_mean,self.base_percent_std)
 self.age_mean = data['Age'].mean()
self.age_std = data['Age'].std()
data['Age'] = standardisation(data['Age'],self.age_mean,self.age_std)
 self.weeks_mean = data['Weeks'].mean()
self.weeks_std = data['Weeks'].std()
data['Weeks'] = standardisation(data['Weeks'],self.weeks_mean,self.weeks_std)
 if self.normalization:
self.base_week_min = data['Base_week'].min()
self.base_week_max = data['Base_week'].max()
data['Base_week'] = normalization(data['Base_week'],self.base_week_max,self.base_week_min)
 self.base_fvc_min = data['Base_FVC'].min()
self.base_fvc_max = data['Base_FVC'].max()
data['Base_FVC'] = normalization(data['Base_FVC'],self.base_fvc_max,self.base_fvc_min)
 self.base_percent_min = data['Percent'].min()
self.base_percent_max = data['Percent'].max()
data['Percent'] = normalization(data['Percent'],self.base_percent_max,self.base_percent_min)
 self.age_min = data['Age'].min()
self.age_max = data['Age'].max()
data['Age'] = normalization(data['Age'],self.age_max,self.age_min)
 self.weeks_min = data['Weeks'].min()
self.weeks_max = data['Weeks'].max()
data['Weeks'] = normalization(data['Weeks'],self.weeks_max,self.weeks_min)
 self.base_week_min = data['Min_week'].min()
self.base_week_max = data['Min_week'].max()
data['Min_week'] = normalization(data['Min_week'],self.base_week_max,self.base_week_min)
 return data<load_pretrained>
y = train['SalePrice']
train.drop("SalePrice", axis = 1, inplace = True )
pickefile = open('/kaggle/input/data-preparation-for-osic/data_prep', 'rb')
 data_prep = pickle.load(pickefile)
 pickefile.close()<categorify>
ntrain = train.shape[0]
all_data = pd.concat(( train, test)).reset_index(drop=True)
 print("all_data size is : {}".format(all_data.shape))
def ordinal_encode(df, col, order_list):
df[col] = df[col].astype('category', ordered=True, categories=order_list ).cat.codes
return df
def label_encode(df, col):
for c in col:
encoder = preprocessing.LabelEncoder()
df[c] = encoder.fit_transform(df[c].astype(str))
return df
def split_all_data(all_data, ntrain):
print(( 'Split all_data back to train and test: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 train_df = all_data[:ntrain]
test_df = all_data[ntrain:]
return train_df, test_df

 print(( 'Ordinal Encoding: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 order_list = ['Ex', 'Gd', 'TA', 'Fa', 'Po']
cols = ['KitchenQual', 'ExterQual', 'ExterCond', 'HeatingQC']
for col in cols:
all_data = ordinal_encode(all_data, col, order_list)
 order_list = ['Ex', 'Gd', 'TA', 'Fa', 'Po', 'NA']
cols = ['BsmtQual', 'BsmtCond']
for col in cols:
all_data = ordinal_encode(all_data, col, order_list)
 order_list = ['Gd', 'Av', 'Mn', 'No', 'NA']
cols = ['BsmtExposure', 'FireplaceQu', 'GarageQual', 'GarageCond']
for col in cols:
all_data = ordinal_encode(all_data, col, order_list)
 order_list = ['Typ', 'Min1', 'Min2', 'Mod', 'Maj1', 'Maj2', 'Sev', 'Sal']
cols = ['Functional']
for col in cols:
all_data = ordinal_encode(all_data, col, order_list)
 order_list = ['Fin', 'RFn', 'Unf', 'NA']
cols = ['GarageFinish']
for col in cols:
all_data = ordinal_encode(all_data, col, order_list)
 order_list = ['Ex', 'Gd', 'TA', 'Fa', 'NA']
cols = ['PoolQC']
for col in cols:
all_data = ordinal_encode(all_data, col, order_list)

print(( 'Label Encoding: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 cols_to_label_encode = ['MSSubClass', 'MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2',
'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'Foundation', 'BsmtFinType1', 'BsmtFinType2', 'Heating', 'CentralAir',
'Electrical', 'GarageType', 'PavedDrive', 'Fence', 'MiscFeature', 'SaleType', 'SaleCondition']
all_data = label_encode(all_data, cols_to_label_encode)
 train, test = split_all_data(all_data, ntrain)
 print("Train Shape: " + str(train.shape))
print("Test Shape: " + str(test.shape))
X_prediction = data_prep(X_prediction)
<prepare_x_and_y>
def score_transformer(y):
y = np.log(y)
 return y
y = score_transformer(y )
<load_pretrained>
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data['fe.sum.GrLivArea_BsmtFinSF1_BsmtFinSF2'] = all_data['GrLivArea'] + all_data['BsmtFinSF1'] + all_data['BsmtFinSF2']
all_data['fe.sum.OverallQual_Overall_Cond'] = all_data['OverallQual'] + all_data['OverallCond']
all_data['fe.mult.OverallQual_Overall_Cond'] = all_data['OverallQual'] * all_data['OverallCond']
all_data['fe.sum.KitchenQual_ExterQual'] = all_data['KitchenQual'] + all_data['ExterQual']
all_data['fe.mult.OverallQual_Overall_Cond'] = all_data['OverallQual'] * all_data['OverallCond']
all_data['fe.ratio.1stFlrSF_2ndFlrSF'] = all_data['1stFlrSF'] / all_data['2ndFlrSF']
all_data['fe.ratio.BedroomAbvGr_GrLivArea'] = all_data['BedroomAbvGr'] / all_data['GrLivArea']
pickefile = open('/kaggle/input/trained-cnn-mlp-for-osic/list_patient_score', 'rb')
 list_patient_score = pickle.load(pickefile)
 pickefile.close()<feature_engineering>
train_features = list(all_data)
 train_features = [e for e in train_features if e not in('ExterQual', 'Condition2', 'GarageCond', 'Street', 'Alley', 'PoolArea', 'PoolQC', 'Utilities',
'GarageQual', 'MiscVal', 'MiscFeature')]
train, test = split_all_data(all_data, ntrain)
 train_features.remove('Id')
list_patient_weight = []
for i in list_patient_score:
if 6.4>np.mean(i[1]):
list_patient_weight.append([i[0], 3])
 elif 6.8>np.mean(i[1])>6.4:
list_patient_weight.append([i[0], 50])
 elif 7.1>np.mean(i[1])>6.8:
list_patient_weight.append([i[0], 100])
 elif 50>np.mean(i[1])>7.6:
list_patient_weight.append([i[0], 3])
 else:
list_patient_weight.append([i[0], 3])
 train['Weight'] = train.Patient.map(dict(list_patient_weight))<load_pretrained>
nfolds=5
kf = KFold(n_splits=nfolds, shuffle=True, random_state=37)
 y_valid_pred = 0*y
y_valid_pred_cat = 0*y
fold_scores = [0] * nfolds
fold_scores_cat = [0] * nfolds
importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 sub_reg_preds_cat = np.zeros(test.shape[0])
 for fold_,(train_index, val_index)in enumerate(kf.split(train, y)) :
trn_x, trn_y = train[train_features].iloc[train_index], y.iloc[train_index]
val_x, val_y = train[train_features].iloc[val_index], y.iloc[val_index]
reg = LGBMRegressor(
 num_leaves=15,
max_depth=3,
min_child_weight=50,
learning_rate=0.04,
n_estimators=1000,
reg_alpha=0.01,
reg_lambda=5,
subsample=1,
colsample_bytree=0.21,
random_state=2 
)
 reg.fit(
 trn_x, trn_y,
eval_set=[(val_x, val_y)],
early_stopping_rounds=20,
verbose=100,
eval_metric='rmse' 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 y_valid_pred.iloc[val_index] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 y_valid_pred[y_valid_pred < 0] = 0
fold_score = reg.best_score_['valid_0']['rmse']
fold_scores[fold_] = fold_score
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += _preds / nfolds
print("LightGBM CV RMSE: " + str(mean_squared_error(y, y_valid_pred)**.5))
print("LightGBM CV standard deviation: " + str(np.std(fold_scores)))
<feature_engineering><EOS> 
sub_reg_preds = np.exp(sub_reg_preds)
 test.is_copy = False
test.loc[:,'SalePrice'] = sub_reg_preds
test[['Id', 'SalePrice']].to_csv("tutorial_sub.csv", float_format='%.8f', index=False)