Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<prepare_x_and_y>
warnings.simplefilter(action='ignore', category=FutureWarning )
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str )<feature_engineering>
DF = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 Test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )
for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >=0.5, True, False )<string_transform>
DF['LotFrontage']=DF.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
DF['GarageYrBlt']=DF.groupby('Neighborhood')['GarageYrBlt'].transform(lambda x: x.fillna(x.median()))
DF['MasVnrArea']=DF.groupby('Neighborhood')['MasVnrArea'].transform(lambda x: x.fillna(x.median()))
tokenizer = text.Tokenizer(filters = CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test )<concatenate>
Test['LotFrontage']=Test.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
Test['GarageYrBlt']=Test.groupby('Neighborhood')['GarageYrBlt'].transform(lambda x: x.fillna(x.median()))
Test['MasVnrArea']=Test.groupby('Neighborhood')['MasVnrArea'].transform(lambda x: x.fillna(x.median()))
x_train = sequence.pad_sequences(x_train,maxlen=MAX_LEN)
 x_test= sequence.pad_sequences(x_test, maxlen=MAX_LEN)
<define_variables>
DF['cond*qual'] =(DF['OverallCond'] * DF['OverallQual'])/ 100.0
DF['home_age_when_sold'] = DF['YrSold'] - DF['YearBuilt']
DF['garage_age_when_sold'] = DF['YrSold'] - DF['GarageYrBlt']
DF['TotalSF'] = DF['TotalBsmtSF'] + DF['1stFlrSF'] + DF['2ndFlrSF']
DF['total_porch_area'] = DF['WoodDeckSF'] + DF['OpenPorchSF'] + DF['EnclosedPorch'] + DF['3SsnPorch'] + DF['ScreenPorch']
DF['Totalsqrfootage'] =(DF['BsmtFinSF1'] + DF['BsmtFinSF2'] + DF['1stFlrSF'] + DF['2ndFlrSF'])
 DF['Total_Bathrooms'] =(DF['FullBath'] +(0.5 * DF['HalfBath'])+ DF['BsmtFullBath'] +(0.5 * DF['BsmtHalfBath']))
sample_weights = np.ones(len(train_df), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN]*(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])*train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights/=sample_weights.mean()<define_variables>
Test['cond*qual'] =(Test['OverallCond'] * Test['OverallQual'])/ 100.0
Test['home_age_when_sold'] = Test['YrSold'] - Test['YearBuilt']
Test['garage_age_when_sold'] = Test['YrSold'] - Test['GarageYrBlt']
Test['TotalSF'] = Test['TotalBsmtSF'] + Test['1stFlrSF'] + Test['2ndFlrSF']
Test['total_porch_area'] = Test['WoodDeckSF'] + Test['OpenPorchSF'] + Test['EnclosedPorch'] + Test['3SsnPorch'] + Test['ScreenPorch']
Test['Totalsqrfootage'] =(Test['BsmtFinSF1'] + Test['BsmtFinSF2'] + Test['1stFlrSF'] + Test['2ndFlrSF'])
 Test['Total_Bathrooms'] =(Test['FullBath'] +(0.5 * Test['HalfBath'])+ Test['BsmtFullBath'] +(0.5 * Test['BsmtHalfBath']))
embedding_matrix = np.concatenate([build_matrix(tokenizer.word_index,f)for f in EMBEDDING_FILES], axis =-1)
 del train_df, tokenizer
gc.collect()
checkpoint_predictions = []
weights = []<train_model>
Old_Cols=['OverallCond','OverallQual','YrSold','YearBuilt','YrSold','GarageYrBlt','TotalBsmtSF','1stFlrSF','2ndFlrSF','WoodDeckSF','OpenPorchSF','EnclosedPorch','3SsnPorch','ScreenPorch','BsmtFinSF1','BsmtFinSF2','1stFlrSF','2ndFlrSF','FullBath','HalfBath','BsmtFullBath','BsmtHalfBath']
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=2,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks = [
LearningRateScheduler(lambda _: 1e-3*(0.55**global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=BATCH_SIZE)[0].flatten())
 weights.append(2 ** global_epoch)
 del model
gc.collect()<set_options>
Final_cols=[]
for i in DF.columns:
if i not in Old_Cols and i!='SalePrice':
Final_cols.append(i)
 PF=DF[Final_cols]
gc.collect()<save_to_csv>
Final_cols=[]
for i in Test.columns:
if i not in Old_Cols and i!='SalePrice':
Final_cols.append(i)
 TF=Test[Final_cols]
predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
 submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions})
 submission.to_csv('submission.csv', index=False )<import_modules>
y = DF.SalePrice
import numpy as np
import pandas as pd
import os
import time
import gc
import random
from tqdm._tqdm_notebook import tqdm_notebook as tqdm
from keras.preprocessing import text, sequence
import torch
from torch import nn
from torch.utils import data
from torch.nn import functional as F
from fastai.train import Learner
from fastai.train import DataBunch
from fastai.callbacks import *
from fastai.basic_data import DatasetType<set_options>
X_train, X_test, y_train, y_test = train_test_split(PF, y, test_size=0.3 )
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<define_variables>
def Change(x):
for col in x.select_dtypes(include=['object'] ).columns:
x[col] = x[col].astype('category')
 for col in x.select_dtypes(include=['category'] ).columns:
x[col] = x[col].cat.codes
return x
CRAWL_EMBEDDING_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
GLOVE_EMBEDDING_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220<compute_test_metric>
X_train = Change(X_train)
 X_test = Change(X_test )
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))<categorify>
model = XGBRegressor(colsample_bytree=1,
gamma=0.5,
learning_rate=0.005,
max_depth=9,
min_child_weight=1.5,
n_estimators=5000,
reg_alpha=0.4,
reg_lambda=0.45,
subsample=0.6,
seed=42 )
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words<compute_test_metric>
model.fit(X_train, y_train)
 model.score(X_test,y_test)*100
def sigmoid(x):
return 1/(1 + np.exp(-x))<train_model>
SalePrice = pd.DataFrame(model.predict(Change(TF)))
 Id = pd.DataFrame(TF.Id)
 result = pd.concat([Id, SalePrice], axis=1)
 result.columns = ['Id', 'SalePrice']
def train_model(learn,test,output_dim,lr=0.00083,
batch_size=512, n_epochs=4,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(1)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds<normalization>
result.to_csv('submission.csv',index=False )