Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<categorify>
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
def tf_encode(id_num, keyword, lang, target):
lang, target = tf.py_function(
 text_encode,
[keyword, lang, target],
[tf.int64, tf.int64])
 id_num.set_shape(None)
 lang.set_shape([None])
 target.set_shape([])
 return id_num, lang, target<choose_model_class>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
class EmbeddingLayer(object):
def __init__(self):
self.kernels = tf.Variable(initial_value=embedding_matrix, trainable=False, name='Embedding_kernels')
 def __call__(self, x):
embeddings = tf.nn.embedding_lookup(params=self.kernels, ids=x)
 return embeddings<normalization>
print('Train shape: ', train.shape)
 print('Test Shape: ', test.shape)
 print('Train:
', train.dtypes.value_counts())
 print('Test:
', test.dtypes.value_counts() )
def get_angles(pos, i, d_model):
angle_rates = 1 / np.power(10000,(2*(i//2)) /np.float32(d_model))
return pos * angle_rates<categorify>
merged_df = pd.concat([train, test], axis = 0, sort = True)
 print('Merged Shape:
', merged_df.shape)
 print('Merged :
', merged_df.dtypes.value_counts())
def positional_encoding(postion, d_model):
angle_rads = get_angles(np.arange(postion)[:,np.newaxis], np.arange(d_model)[np.newaxis,:], d_model)
 angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])
 angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])
 pos_encoding = angle_rads[np.newaxis,...]
return tf.cast(pos_encoding, dtype=tf.float32 )<categorify>
merged_df.loc[:, ['MSSubClass', 'OverallQual', 'OverallCond']] = merged_df.loc[:, ['MSSubClass', 'OverallQual', 'OverallCond']].astype('object')
 print('New Shape of merged data:
', merged_df.shape)
 print('Merged structure :
', merged_df.dtypes.value_counts() )
def create_padding_mask(seq):
seq = tf.cast(tf.math.equal(seq, 0), dtype=tf.float32)
 return seq[:, tf.newaxis, tf.newaxis, :]<categorify>
train.drop(train[train.GrLivArea>4000].index, inplace = True)
 train.reset_index(drop = True, inplace = True)
 display(train.shape )
def create_look_ahead_mask(size):
mask = 1 - tf.linalg.band_part(tf.ones(( size,size)) , -1, 0)
 return mask<normalization>
train.drop(train[train.TotalBsmtSF>3000].index, inplace = True)
 train.reset_index(drop = True, inplace = True )
def scaled_dot_product_attention(q, k, v, mask):
matmul_qk = tf.matmul(q, k, transpose_b=True)
 dk = tf.cast(tf.shape(q)[-1], dtype=tf.float32)
 scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
 if mask is not None:
scaled_attention_logits +=(mask * -1e9)
 attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)
 output = tf.matmul(attention_weights, v)
 return output, attention_weights<choose_model_class>
y_train = train.SalePrice
train.drop('SalePrice', axis = 1, inplace = True)
 train.shape
merged_df = pd.concat([train, test], axis = 0)
 merged_df.shape
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, num_heads, d_model):
super(MultiHeadAttention, self ).__init__()
self.num_heads = num_heads
self.d_model = d_model
assert d_model % num_heads == 0
self.depth = d_model // num_heads
self.wq = tf.keras.layers.Dense(units=d_model)
 self.wk = tf.keras.layers.Dense(units=d_model)
 self.wv = tf.keras.layers.Dense(units=d_model)
 self.dense = tf.keras.layers.Dense(units=d_model)
 def split_heads(self, x, batch_size):
x = tf.reshape(x,(batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
 def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
q = self.wq(q)
 k = self.wk(k)
 v = self.wv(v)
 q = self.split_heads(q, batch_size)
 k = self.split_heads(k, batch_size)
 v = self.split_heads(v, batch_size)
 scaled_attention, attention_weights = scaled_dot_product_attention(q, k, v, mask)
 scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])
 concat_attention = tf.reshape(scaled_attention,(batch_size, -1, self.d_model))
output = self.dense(concat_attention)
 return output, attention_weights<choose_model_class>
merged_df.loc[:, ['MSSubClass', 'OverallQual', 'OverallCond']] = merged_df.loc[:, ['MSSubClass', 'OverallQual', 'OverallCond']].astype('object')
 print('New Shape of merged data:
', merged_df.shape)
 print('Merged structure :
', merged_df.dtypes.value_counts() )
def point_wise_feed_forward_network(d_model, dff):
return tf.keras.Sequential([
tf.keras.layers.Dense(units=dff, activation='relu'),
tf.keras.layers.Dense(units=d_model)
 ] )<choose_model_class>
missing_col = len(merged_df)- merged_df.loc[:, np.sum(merged_df.isnull())>0].count()
missing_col
class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate):
super(EncoderLayer, self ).__init__()
self.mha = MultiHeadAttention(num_heads=num_heads, d_model=d_model)
 self.ffn = point_wise_feed_forward_network(d_model=d_model, dff=dff)
 self.dropout1 = tf.keras.layers.Dropout(rate=rate)
 self.dropout2 = tf.keras.layers.Dropout(rate=rate)
 self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 def call(self, x, training, mask):
attn_output, attn_weights = self.mha(x, x, x, mask)
 attn_output = self.dropout1(attn_output, training=training)
 out1 = self.layernorm1(x + attn_output)
 ffn_output = self.ffn(out1)
 ffn_output = self.dropout2(ffn_output, training=training)
 out2 = self.layernorm2(out1 + ffn_output)
 return out2<categorify>
replace_none = merged_df.loc[:, ['PoolQC','Fence','MiscFeature','GarageCond','GarageQual','GarageFinish','GarageType','Alley','FireplaceQu','BsmtFinType2','BsmtFinType1','BsmtExposure','BsmtCond','BsmtQual']]
for i in replace_none.columns:
merged_df[i].fillna('None', inplace = True)
 missing_col = len(merged_df)- merged_df.loc[:, np.sum(merged_df.isnull())>0].count()
missing_col
class Encoder(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff, maximum_position_encoding, rate=0.1):
super(Encoder, self ).__init__()
self.num_layers = num_layers
self.d_model = d_model
self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)
 self.enc_layers = [EncoderLayer(d_model=d_model, num_heads=num_heads, dff=dff, rate=rate)for _ in range(num_layers)]
self.dropout = tf.keras.layers.Dropout(rate=rate)
 def call(self, x, training, mask, positition=True):
seq_len = tf.shape(x)[1]
x *= tf.math.sqrt(tf.cast(self.d_model, dtype=tf.float32))
if positition:
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
 for i in range(self.num_layers):
x = self.enc_layers[i](x, training, mask)
 return x<choose_model_class>
replace_mode = merged_df.loc[:, ['MSZoning', 'Electrical', 'Utilities', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'KitchenQual', 'Functional', 'SaleType']]
for i in replace_mode.columns:
merged_df[i].fillna(merged_df[i].mode() [0], inplace = True)
 missing_col = len(merged_df)- merged_df.loc[:, np.sum(merged_df.isnull())>0].count()
missing_col
class OutputLayer(tf.keras.layers.Layer):
def __init__(self, units, rate):
super(OutputLayer, self ).__init__()
self.gapool1d = tf.keras.layers.GlobalAveragePooling1D()
self.dense = tf.keras.layers.Dense(units=units, activation='relu')
 self.final_layer = tf.keras.layers.Dense(units=2)
 self.dropout = tf.keras.layers.Dropout(rate=rate)
 def call(self, enc, training):
x = self.gapool1d(enc)
 x = self.dense(x)
 x = self.dropout(x, training=training)
 x = self.final_layer(x)
 return x<categorify>
replace_mode = merged_df.loc[:, ['LotFrontage','MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath', 'GarageYrBlt', 'GarageCars', 'GarageArea']]
for i in replace_mode.columns:
merged_df[i].fillna(merged_df[i].median() , inplace = True)
 missing_col = len(merged_df)- merged_df.loc[:, np.sum(merged_df.isnull())>0].count()
missing_col
class TransformerCategorical(tf.keras.Model):
def __init__(self, num_layers, d_model, num_heads, dff, maximum_position_encoding, output_units, rate=0.1):
super(TransformerCategorical, self ).__init__()
self.embedding = EmbeddingLayer()
self.encoder = Encoder(num_layers, d_model, num_heads, dff, maximum_position_encoding, rate)
 self.outputlayer = OutputLayer(output_units, rate)
 def call(self, lang, training, enc_padding_mask):
enc_input = self.embedding(lang)
 enc_output = self.encoder(enc_input, training, enc_padding_mask)
 out = self.outputlayer(enc_output, training)
 return out<choose_model_class>
numeric_skew = pd.DataFrame(data = merged_df.select_dtypes(include = ['int64', 'float64'] ).skew() , columns = ['Skewness'])
 numeric_skew_sort = numeric_skew.sort_values(ascending = False, by = 'Skewness')
 numeric_skew_sort
num_layers = 6
d_model = d_model
num_heads = 6
dff = 512
pe_input = input_vocab_size
output_units = 64
rate = 0.1<choose_model_class>
standard_scaler = StandardScaler()
merged_df_num_scaled = standard_scaler.fit_transform(merged_df_num)
 merged_df_num_scaled = pd.DataFrame(data = merged_df_num_scaled, columns = merged_df_num.columns, index = merged_df_num.index )
tsfr_categorical = TransformerCategorical(num_layers, d_model, num_heads, dff, pe_input, output_units, rate )<normalization>
merged_df_cat = merged_df.select_dtypes(include = ['object'] ).astype('category')
 merged_df_cat.shape
merged_df_cat_encoded = merged_df_cat.apply(LabelEncoder().fit_transform)
 merged_df_cat_encoded.shape
merged_df_one_hot = merged_df_cat.select_dtypes(include = ['category'])
 merged_df_one_hot = pd.get_dummies(merged_df_one_hot, drop_first = True)
 merged_df_one_hot.shape
merged_df_cat_encoded_final = pd.concat([merged_df_one_hot, merged_df_cat_encoded], axis = 1)
 merged_df_final_processed =pd.concat([merged_df_num_scaled, merged_df_cat_encoded_final], axis = 1)
 merged_df_final_processed.shape
class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):
def __init__(self, d_model, warmup_steps=600):
super(CustomSchedule, self ).__init__()
self.d_model = tf.cast(d_model, dtype=tf.float32)
 self.warmup_steps = warmup_steps
def __call__(self, step):
step = step + 100
arg1 = step ** -0.8
arg2 = step *(self.warmup_steps ** -1.5)
 return tf.math.rsqrt(self.d_model)* tf.math.minimum(arg1, arg2 )<choose_model_class>
train_final = merged_df_final_processed.iloc[0:1438, :]
test_final = merged_df_final_processed.iloc[1438:, :]
y_train = y_train
X = train_final
y = y_train
learning_rate = CustomSchedule(d_model)
 optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9 )<compute_test_metric>
state = 40
kfolds = KFold(n_splits=10, shuffle=True, random_state=state)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse )
loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)
 def loss_function(real, pred):
loss_ = loss_object(real, pred)
 return tf.reduce_mean(loss_)
 def acc_function(real, pred):
predictions = tf.math.argmax(pred, axis=1)
 predictions = tf.cast(predictions, dtype=tf.int64)
 accuracy = tf.cast(tf.math.equal(predictions, real), dtype=tf.float32)
 ave_acc = tf.reduce_mean(accuracy)
 return ave_acc<data_type_conversions>
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
train_step_signature = [
tf.TensorSpec(shape=(None, None), dtype=tf.int64),
tf.TensorSpec(shape=(None,), dtype=tf.int64)
 ]<categorify>
ridge = RidgeCV(alphas=alphas_alt, cv=kfolds)
 lasso = LassoCV(max_iter=1e7, alphas=alphas2, random_state=42, cv=kfolds)
 elasticnet = ElasticNetCV(max_iter=1e7, alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio)
 svr = SVR(C= 20, epsilon= 0.008, gamma=0.0003 )
@tf.function(input_signature=train_step_signature)
 def train_step(lang, targ):
enc_padding_mask = create_padding_mask(lang)
 with tf.GradientTape() as tape:
predictions = tsfr_categorical(lang, True, enc_padding_mask)
 loss = loss_function(targ, predictions)
 loss_regularization = []
for w in tsfr_categorical.trainable_variables:
loss_regularization.append(tf.nn.l2_loss(w))
loss_regularization = tf.reduce_sum(tf.stack(loss_regularization))
loss = loss + 0.01 * loss_regularization
gradients = tape.gradient(loss, tsfr_categorical.trainable_variables)
 optimizer.apply_gradients(zip(gradients, tsfr_categorical.trainable_variables))
accuracy = acc_function(targ, predictions)
 return loss, accuracy<compute_test_metric>
gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state =42 )
@tf.function(input_signature=train_step_signature)
 def valid_step(lang, targ):
enc_padding_mask = create_padding_mask(lang)
 predictions = tsfr_categorical(lang, False, enc_padding_mask)
 loss = loss_function(targ, predictions)
 accuracy = acc_function(targ, predictions)
 return loss, accuracy<categorify>
lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1,
)
BATCH_SIZE = 2048
BUFFLE_SIZE = 8000
def data_generator(data):
dataset = tf.data.Dataset.from_tensor_slices(( data['id'], data['keyword'], data['text'], data['target']))
dataset = dataset.map(tf_encode)
 dataset = dataset.cache().shuffle(BUFFLE_SIZE ).padded_batch(BATCH_SIZE)
 dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
 return dataset
train_data = train.sample(frac=0.9)
 val_data = train[~train.index.isin(train_data.index)]
train_dataset = data_generator(train_data)
 val_dataset = data_generator(val_data)
 sample = next(iter(train_dataset))
print(sample )<define_variables>
xgboost = XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006 )
Epochs = 100<find_best_params>
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet, gbr, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True )
tensorboard = {'Train_loss':[],'Train_acc':[],'Val_loss':[],'Val_acc':[]}
for epoch in range(Epochs):
train_loss = []
train_accuracy = []
val_loss = []
val_accuracy = []
for _, lang, targ in train_dataset:
loss, acc = train_step(lang, targ)
 train_loss.append(loss)
 train_accuracy.append(acc)
 for _, lang, targ in val_dataset:
loss, acc = valid_step(lang, targ)
 val_loss.append(loss)
 val_accuracy.append(acc)
 ave_val_acc = np.mean(val_accuracy)
 if epoch !=0 and max(tensorboard['Val_acc'])<= ave_val_acc:
tsfr_categorical.save_weights('/kaggle/working/checkpoint/best_val')
 tensorboard['Train_loss'].append(np.mean(train_loss))
tensorboard['Train_acc'].append(np.mean(train_accuracy))
tensorboard['Val_loss'].append(np.mean(val_loss))
tensorboard['Val_acc'].append(ave_val_acc )<compute_test_metric>
score = cv_rmse(ridge)
 score = cv_rmse(lasso)
 print("LASSO: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(elasticnet)
 print("elastic net: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(svr)
 print("SVR: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(gbr)
 print("gbr: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(xgboost)
 print("xgboost: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
for index in range(len(tensorboard['Train_loss'])) :
print(f"\033[0;34mEpoch\033[0m:{index}, Loss:{tensorboard['Train_loss'][index]}, Accuracy:{tensorboard['Train_acc'][index]}",
f"Valid_Loss:{tensorboard['Val_loss'][index]}, Valid_Accuracy:{tensorboard['Val_acc'][index]}" )<load_from_csv>
print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(X), np.array(y))
print('elasticnet')
 elastic_model_full_data = elasticnet.fit(X, y)
 print('Lasso')
 lasso_model_full_data = lasso.fit(X, y)
 print('Ridge')
 ridge_model_full_data = ridge.fit(X, y)
 print('Svr')
 svr_model_full_data = svr.fit(X, y)
 print('GradientBoosting')
 gbr_model_full_data = gbr.fit(X, y)
 print('xgboost')
 xgb_model_full_data = xgboost.fit(X, y)
 print('lightgbm')
 lgb_model_full_data = lightgbm.fit(X, y )
venv_target = np.array([0]*len(test['text']))
test_target = pd.read_csv('/kaggle/input/test-twitter/perfect_submission.csv')
 test_dataset = tf.data.Dataset.from_tensor_slices(( test['id'], test['keyword'], test['text'], test_target['target']))
test_dataset = test_dataset.map(tf_encode)
 test_dataset = test_dataset.padded_batch(BATCH_SIZE )<load_pretrained>
def blend_models_predict(X):
return(( 0.1 * elastic_model_full_data.predict(X)) + \ 
(0.05 * lasso_model_full_data.predict(X)) + \ 
(0.1 * ridge_model_full_data.predict(X)) + \ 
(0.1 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.15 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.3 * stack_gen_model.predict(np.array(X))))
evaluate = TransformerCategorical(num_layers, d_model, num_heads, dff, pe_input, output_units, rate)
 evaluate.load_weights('/kaggle/working/checkpoint/best_val' )<categorify>
print('RMSLE score on train data:')
 print(rmsle(y, blend_models_predict(X)) )
results = []
for id_num, lang, _ in test_dataset:
enc_padding_mask = create_padding_mask(lang)
 predictions = evaluate(lang, False, enc_padding_mask)
 predictions = tf.math.argmax(predictions, axis=1)
 predictions = tf.cast(predictions, dtype=tf.int32)
 predictions = tf.reshape(predictions,(-1))
results.extend(zip(id_num.numpy() , predictions.numpy()))<compute_test_metric>
print('Predict submission')
 submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blend_models_predict(test_final)) )
<save_to_csv><EOS> 
submission.head(5)
 submission.to_csv('submission.csv',index=False )