Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<set_options>
print(sys.version )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
import random
import datetime
import numpy as np
import pandas as pd
import seaborn as sns
import xgboost as xgb
import lightgbm as lgb
from pathlib import Path
import matplotlib.cm as cm
import matplotlib.pyplot as plt
from scipy.special import boxcox1p
from sklearn.svm import SVR
from sklearn.kernel_ridge import KernelRidge
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split,KFold, cross_val_score
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import ElasticNet
from sklearn.preprocessing import RobustScaler
from sklearn.ensemble import RandomForestRegressor
from dateutil.relativedelta import relativedelta
model_name = 'jplu/tf-xlm-roberta-large'
n_epochs = 10
max_len = 80
batch_size = 16 * strategy.num_replicas_in_sync<load_from_csv>

 def columns_nan_description(df):
if not isinstance(df,pd.DataFrame):
raise Exception('Please insert a pandas dataframe')
 number_of_rows = len(df)
 number_of_columns = len(df.columns)
 number_of_null_columns = len(df.columns[df.isnull().any() ])
 print(f'There are {number_of_columns} columns and {number_of_null_columns}({number_of_null_columns/number_of_columns*100:.2f}%)of them have null values.')
 for column in df.columns:
if df[column].isnull().any() :
number_of_null_values = df[column].isnull().sum()
print(f'{column} - {number_of_null_values} values are null - {number_of_null_values/number_of_rows*100:.2f}%' )
train = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/train.csv')
 test = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/test.csv')
 submission = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/sample_submission.csv' )<load_pretrained>

 def rmse(predictions, targets):
return np.sqrt(((predictions - targets)** 2 ).mean() )
tokenizer = AutoTokenizer.from_pretrained(model_name )<categorify>
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 train.head(5 )
train_text = train[['premise', 'hypothesis']].values.tolist()
test_text = test[['premise', 'hypothesis']].values.tolist()
train_encoded = tokenizer.batch_encode_plus(
 train_text,
pad_to_max_length=True,
max_length=max_len 
)
 test_encoded = tokenizer.batch_encode_plus(
 test_text,
pad_to_max_length=True,
max_length=max_len
)<split>
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 test.head(5 )
x_train, x_valid, y_train, y_valid = train_test_split(
 train_encoded['input_ids'], train.label.values,
test_size=0.2, random_state=2020 
)
 x_test = test_encoded['input_ids']<split>

 train_null_columns_ovr50 = ['Alley','MiscFeature','Fence','PoolQC','FireplaceQu']
test_null_columns_ovr50 = ['MiscFeature','Fence','PoolQC','FireplaceQu','Alley']
auto = tf.data.experimental.AUTOTUNE
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(batch_size)
.prefetch(auto)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(batch_size)
.cache() 
.prefetch(auto)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(batch_size)
 )<categorify>

 train_no_feature_columns = ['MasVnrType','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','GarageType','GarageFinish','GarageQual','GarageCond']
test_no_feature_columns = ['MasVnrType','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','GarageType','GarageFinish','GarageQual','GarageCond']
with strategy.scope() :
transformer_encoder = TFAutoModel.from_pretrained(model_name)
 input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 sequence_output = transformer_encoder(input_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(3, activation='softmax' )(cls_token)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 model.summary()<train_model>

 for column in train_null_columns_ovr50:
train[column].fillna("NA",inplace=True)
 for column in test_null_columns_ovr50:
test[column].fillna("NA",inplace=True)
 for column in train_no_feature_columns:
train[column].fillna("NA",inplace=True)
 for column in test_no_feature_columns:
test[column].fillna("NA",inplace=True )
n_steps = len(x_train)// batch_size
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=n_epochs
)<predict_on_test>


train['GarageYrBlt'][train['GarageType']=='NA'] = 0
train['GarageYrBlt'][train['GarageYrBlt'].isnull() ] = train['YearBuilt'][train['GarageYrBlt'].isnull() ]
test['GarageYrBlt'][test['GarageType']=='NA'] = 0
test['GarageYrBlt'][test['GarageYrBlt'].isnull() ] = test['YearBuilt'][test['GarageYrBlt'].isnull() ]

 train['MasVnrArea'][train['MasVnrType']=='NA'] = 0
test['MasVnrArea'][test['MasVnrType']=='NA'] = 0

 test['TotalBsmtSF'][test['BsmtQual']=='NA'] = 0
test['GarageArea'][test['GarageArea'].isnull() ]= int(test.loc[test['GarageType']=='Detchd',['GarageArea']].mean())

test['GarageCars'][test['GarageQual']=='NA'] = 0

 test['SaleType'][test['SaleType'].isnull() ] = 'Oth'
test['Functional'][test['Functional'].isnull() ] = 'Typ'

 test['KitchenQual'][test['KitchenQual'].isnull() ] = 'TA'

 test['BsmtHalfBath'][test['BsmtQual'] == 'NA'] = 0

 test['BsmtFullBath'][test['BsmtQual'] == 'NA'] = 0

 test['BsmtUnfSF'][test['BsmtQual'] == 'NA'] = 0

 test['BsmtFinSF2'][test['BsmtQual'] == 'NA'] = 0

 test['BsmtFinSF1'][test['BsmtQual'] == 'NA'] = 0

 test['Exterior2nd'][test['Exterior2nd'].isnull() ] = 'Other'

 test['Exterior1st'][test['Exterior1st'].isnull() ] = 'Other'

 test['Utilities'][test['OverallQual']==6] = 'NoSewr'
test['Utilities'][test['OverallQual']==2] = 'ELO'

 train['Electrical'][train['Electrical'].isnull() ] = 'Mixed'
test_preds = model.predict(test_dataset, verbose=1)
 submission['prediction'] = test_preds.argmax(axis=1 )<save_to_csv>

 def mean_of_neighborhood(df,neighborhood):
mean = df.loc[(df['Neighborhood']==neighborhood),'LotFrontage'].mean()
return mean
train_lot_frontage_mean = {neighborhood:mean_of_neighborhood(train,neighborhood)for neighborhood in list(train['Neighborhood'].unique())}
test_lot_frontage_mean = {neighborhood:mean_of_neighborhood(test,neighborhood)for neighborhood in list(test['Neighborhood'].unique())}
train['LotFrontage'] = train['LotFrontage'].fillna(train['Neighborhood'].map(train_lot_frontage_mean))
test['LotFrontage'] = test['LotFrontage'].fillna(test['Neighborhood'].map(test_lot_frontage_mean))
submission.to_csv('submission.csv', index=False)
 submission.head()<install_modules>

 train['GarageYrBlt'] = train['GarageYrBlt'].astype(int)
 train['HouseAge'] = train.apply(lambda row: years_between(datetime.datetime(row['YearBuilt'],1,1),
datetime.datetime(row['YrSold'],1,1)) if row['YearBuilt'] != 0 else 0,axis=1)
 train['RemodAddAge'] = train.apply(lambda row: years_between(datetime.datetime(row['YearRemodAdd'],1,1),
datetime.datetime(row['YrSold'],1,1)) if row['YearRemodAdd'] != 0 else 0,axis=1)
 train['GarageAge'] = train.apply(lambda row: years_between(datetime.datetime(row['GarageYrBlt'],1,1),
datetime.datetime(row['YrSold'],1,1)) if row['GarageYrBlt'] != 0 else 0,axis=1)
 numerical_columns.extend(['HouseAge','RemodAddAge','GarageAge'])
 excluded_numerical_columns = ['YrSold','MoSold','GarageYrBlt','YearRemodAdd','YearBuilt']
numerical_columns = [x for x in numerical_columns if x not in excluded_numerical_columns]
train[['HouseAge','RemodAddAge','GarageAge']]
!pip install transformers==3.0.2<import_modules>

 test['GarageYrBlt'] = test['GarageYrBlt'].astype(int)
 test['HouseAge'] = test.apply(lambda row: years_between(datetime.datetime(row['YearBuilt'],1,1),
datetime.datetime(row['YrSold'],1,1)) if row['YearBuilt'] != 0 else 0,axis=1)
 test['RemodAddAge'] = test.apply(lambda row: years_between(datetime.datetime(row['YearRemodAdd'],1,1),
datetime.datetime(row['YrSold'],1,1)) if row['YearRemodAdd'] != 0 else 0,axis=1)
 test['GarageAge'] = test.apply(lambda row: years_between(datetime.datetime(row['GarageYrBlt'],1,1),
datetime.datetime(row['YrSold'],1,1)) if row['GarageYrBlt'] != 0 else 0,axis=1)
 test[['HouseAge','RemodAddAge','GarageAge']]
import tensorflow as tf
import transformers
from sklearn.model_selection import KFold<import_modules>

 us_inflation_rates = pd.read_csv(".. /input/us-inflation-rates/US_Inflation_Rates.csv")
 print("US Inflation rates head:")
 print(us_inflation_rates.head())
 print("US Inflation rates tail:")
 print(us_inflation_rates.tail() )
transformers.__version__<import_modules>

 train['SalePriceLog1p'] = np.log1p(train['SalePrice'])
 train['RealSalePriceLog1p'] = np.log1p(train['RealSalePrice'])
 print("Log1p Sale Price skeness: ",train['SalePriceLog1p'].skew())
 print("Log1p Sale Price kurtosis: ",train['SalePriceLog1p'].kurtosis())
 print("Log1p Real Sale Price skeness: ",train['RealSalePriceLog1p'].skew())
 print("Log1p Real Sale Price kurtosis: ",train['RealSalePriceLog1p'].kurtosis() )
from transformers import AutoTokenizer, TFAutoModel<load_from_csv>
def get_cmap(n, name='magma'):

 return plt.cm.get_cmap(name, n )
train_df = pd.read_csv("/kaggle/input/contradictory-my-dear-watson/train.csv")
 test_df = pd.read_csv("/kaggle/input/contradictory-my-dear-watson/test.csv" )<define_variables>
lam = 0.15
for col in numerical_columns_skewness_train.index:
if abs(numerical_columns_skewness_train[col])> 0.75:
train[col] = boxcox1p(train[col],lam)
 for col in numerical_columns_skewness_test.index:
if abs(numerical_columns_skewness_test[col])> 0.75:
test[col] = boxcox1p(test[col],lam )
epochs = 3
maxlen = 50
model_name = "jplu/tf-xlm-roberta-large"
batch_size = 16 * strategy.num_replicas_in_sync<load_pretrained>
train = train.set_index('Id')
 train.head()
tokenizer = AutoTokenizer.from_pretrained(model_name )<define_variables>
test = test.set_index('Id')
 test.head()
%%time
train_encode = tokenizer(list(train_df.premise.values), list(train_df.hypothesis.values),
max_length=maxlen, return_tensors="np", padding=True,
return_token_type_ids=True, return_attention_mask=True )<choose_model_class>
train_sale_price = train['SalePrice']
train_real_sale_price = train['RealSalePrice']
train_sale_price_log1p= train['SalePriceLog1p']
train_real_sale_price_log1p= train['RealSalePriceLog1p']
train = train.drop(['SalePrice','SalePriceLog1p','RealSalePrice','RealSalePriceLog1p'],axis=1)
 train_test = pd.concat([train,test])
 train_test.head()
def get_model(maxlen=50):
base_model = TFAutoModel.from_pretrained(model_name)
 input_ids = tf.keras.layers.Input(shape =(maxlen,), dtype=tf.int32, name="input_ids")
 input_type = tf.keras.layers.Input(shape =(maxlen,), dtype=tf.int32, name="token_type_ids")
 input_mask = tf.keras.layers.Input(shape =(maxlen,), dtype=tf.int32, name="attention_mask")
 embedding = base_model([input_ids, input_mask, input_type])[0]
print(embedding.shape)
 output = tf.keras.layers.Dense(3, activation="softmax" )(embedding[:, 0, :])
 model = tf.keras.models.Model(inputs=[input_ids, input_mask, input_type], outputs = output)
 model.compile(tf.keras.optimizers.Adam(1e-5), "sparse_categorical_crossentropy", ["accuracy"])
 return model<find_best_params>
OH_encoder = OneHotEncoder(drop='first', sparse=False)
 OH_cols = pd.DataFrame(OH_encoder.fit_transform(train_test[categorical_columns]),columns = OH_encoder.get_feature_names(categorical_columns))
OH_cols.index = train_test.index
OH_cols.head(5 )
%%time
ps = cls_model([train_encode['input_ids'][:10], train_encode['attention_mask'][:10], train_encode['token_type_ids'][:10]] )<choose_model_class>
label_columns = ['Utilities','OverallQual','OverallCond','ExterQual','ExterCond','BsmtQual','BsmtExposure',
'BsmtCond','BsmtFinType1','BsmtFinType2','HeatingQC','GarageQual','KitchenQual','FireplaceQu',
'GarageCond','PoolQC']
LB_cols = pd.DataFrame(train_test[label_columns])
 LB_cols = LB_cols.replace(
 {'Utilities':{'AllPub':4,'NoSewr':3,'NoSeWa':2,'ELO':1},
'ExterQual':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1},
'ExterCond':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1},
'BsmtQual':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'NA':0},
'BsmtExposure':{'Gd':4,'Av':3,'Mn':2,'No':1,'NA':0},
'BsmtCond':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'NA':0},
'BsmtFinType1':{'GLQ':6,'ALQ':5,'BLQ':4,'Rec':3,'LwQ':2,'Unf':1,'NA':0},
'BsmtFinType2':{'GLQ':6,'ALQ':5,'BLQ':4,'Rec':3,'LwQ':2,'Unf':1,'NA':0},
'HeatingQC':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1},
'GarageQual':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'NA':0},
'KitchenQual':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1},
'FireplaceQu':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'NA':0},
'GarageCond':{'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'NA':0},
'PoolQC':{'Ex':4,'Gd':3,'TA':2,'Fa':1,'NA':0}} 
)
 MinMax_Scaler = MinMaxScaler()
LB_cols = pd.DataFrame(MinMax_Scaler.fit_transform(LB_cols),columns=label_columns)
 LB_cols.index = train_test.index
LB_cols.head()
fold = KFold(n_splits=3, shuffle=True, random_state=108 )<prepare_x_and_y>
MinMax_Scaler = MinMaxScaler()
MM_cols = pd.DataFrame(MinMax_Scaler.fit_transform(train_test[numerical_columns]),columns = [numerical_columns])
 MM_cols.index = train_test.index
MM_cols.head(5 )
%%time
hists = []
models = []
for i,(train_idx, val_idx)in enumerate(fold.split(np.arange(train_df.label.shape[0]))):
print(f"----FOLD: {i+1}----
",train_idx, val_idx)
 x_train = [train_encode['input_ids'][train_idx],
train_encode['attention_mask'][train_idx],
train_encode['token_type_ids'][train_idx]]
y_train = train_df.label.values[train_idx]
x_val = [train_encode['input_ids'][val_idx],
train_encode['attention_mask'][val_idx],
train_encode['token_type_ids'][val_idx]]
y_val = train_df.label.values[val_idx]
hist=cls_model.fit(x_train, y_train,
epochs=epochs,
batch_size = batch_size,
validation_data =(x_val, y_val), 
)
 hists.append(hist)
 gc.collect()<set_options>
train_test_transformed = OH_cols.join(LB_cols)
 train_test_transformed = train_test_transformed.join(MM_cols)
 train_test_transformed.head(5 )
gc.collect()<string_transform>
test_data_X = train_test_transformed.loc[test.index.tolist() ,:]
train_data_X = train_test_transformed.loc[train.index.tolist() ,:]
%%time
test_encode = tokenizer(list(test_df.premise.values), list(test_df.hypothesis.values),
max_length=maxlen, return_tensors="tf", padding=True,
return_token_type_ids=True, return_attention_mask=True )<predict_on_test>

 lasso = make_pipeline(RobustScaler() , Lasso(alpha=0.002631578947368421, max_iter=50, tol=0.01))
KRR = KernelRidge(alpha=0.7, coef0=10.0, degree=2, kernel='polynomial')
 ENet = make_pipeline(RobustScaler() ,ElasticNet(alpha=0.002380952380952381, l1_ratio=0.2, max_iter=150, tol=1e-05))
model_lgb = lgb.LGBMRegressor(learning_rate=0.07, n_estimators=810, num_leaves=5,objective='regression')
 rfr = RandomForestRegressor(bootstrap=False, max_depth=105, max_features='sqrt',
min_samples_leaf=3, min_samples_split=5,
n_estimators=400)
 model_xgb = xgb.XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.07, max_delta_step=0, max_depth=3,
min_child_weight=1, monotone_constraints='() ',
n_estimators=450, n_jobs=0, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,
tree_method='exact', validate_parameters=1, verbosity=None )
ps = cls_model.predict([test_encode['input_ids'], test_encode['attention_mask'], test_encode['token_type_ids']],
verbose=1, batch_size=batch_size )<create_dataframe>
def rmsle_cv(model,X,y,n_folds=5):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(X.values)
 rmse= np.sqrt(-cross_val_score(model, X.values, y, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
submission = test_df.id.copy().to_frame()
submission['prediction'] = np.argmax(ps, 1 )<save_to_csv>
print(f'Lasso mean RMSE:{rmsle_cv(lasso,train_data_X,train_sale_price_log1p ).mean() }')
 print(f'Kernel Ridge mean RMSE:{rmsle_cv(KRR,train_data_X,train_sale_price_log1p ).mean() }')
 print(f'Elastic Net mean RMSE:{rmsle_cv(ENet,train_data_X,train_sale_price_log1p ).mean() }')
 print(f'LightGBM mean RMSE:{rmsle_cv(model_lgb,train_data_X,train_sale_price_log1p ).mean() }')
 print(f'Random Forest mean RMSE:{rmsle_cv(rfr,train_data_X,train_sale_price_log1p ).mean() }')
 print(f'XGBoost mean RMSE:{rmsle_cv(model_xgb,train_data_X,train_sale_price_log1p ).mean() }' )
submission.to_csv("submission.csv", index = False )<install_modules>
class AverageStackingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
!pip install -q efficientnet<import_modules>
averaged_models = AverageStackingModels(models =(ENet,model_lgb,model_xgb))
print(f'Averaged models mean RMSE:{rmsle_cv(averaged_models,train_data_X,train_sale_price_log1p ).mean() }' )
<set_options><EOS> 
averaged_models.fit(train_data_X.values, train_sale_price_log1p)
 pred_test = averaged_models.predict(test_data_X.values)
 pred_test = np.expm1(pred_test)
 sample_submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 sample_submission['SalePrice'] = pred_test
sample_submission.to_csv('submission.csv',index=False )