Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<define_variables>
plt.style.use('classic')
 %matplotlib inline
sb.set()
warnings.filterwarnings('ignore' )
MAX_LEN = 40<count_unique_values>
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train_len = len(train)
 data_all = train.append(test, sort=False, ignore_index=True)
 features = data_all.shape[1]
idUnique = len(set(data_all.Id))
idTotal = data_all.shape[0]
idDupli = idTotal - idUnique
print("There are " + str(idDupli)+ " duplicate IDs for " + str(idTotal)+ " total entries")
 print("There are " + str(features)+ " variables for " + str(idTotal)+ " total entries" )
qids = pd.Series(list(train_df['qid1'])+ list(train_df['qid2']))
print('Unique Questions number: {}
'.format(len(np.unique(qids))))
q_vals=qids.value_counts() [0:5]
print('Top 5 most frequently asked questions: ')
 for pair in q_vals.iteritems() :
print(train_df.loc[train_df['qid2']==pair[0]]['question1'].head(1 ).values + " count: " + str(pair[1]))
q_vals=q_vals.values<count_duplicates>
quantity = [f for f in data_all.columns if data_all.dtypes[f] != 'object']
quantity.remove('SalePrice')
 quantity.remove('Id')
 quality = [f for f in data_all.columns if data_all.dtypes[f] == 'object']
duplicate_rows = train_df[train_df.duplicated(['qid1','qid2'])]
print("Number of duplicate questions : ", len(duplicate_rows))<feature_engineering>
missing_quant =(data_all[quantity].isnull().sum() /data_all[quantity].isnull().count() ).sort_values(ascending=False)
 missing_quant = missing_quant[missing_quant > 0] * 100
print("There are {} quantitative features with missing values :".format(missing_quant.shape[0]))
missing_quant = pd.DataFrame({'Percent' :missing_quant})
 missing_quant.head()
def common_words(row):
q1_word_set = set(map(lambda word: word.lower().strip() , row['question1'].split(" ")))
 q2_word_set = set(map(lambda word: word.lower().strip() , row['question2'].split(" ")))
 return 1.0 * len(q1_word_set & q2_word_set)
 train_df['common_words'] = train_df.apply(common_words, axis=1 )<feature_engineering>
np.where(pd.isnull(data_all.GarageArea))
def shared_words(row):
q1_word_set = set(map(lambda word: word.lower().strip() , row['question1'].split(" ")))
 q2_word_set = set(map(lambda word: word.lower().strip() , row['question2'].split(" ")))
 return 1.0 * len(q1_word_set & q2_word_set)/(len(q1_word_set)+ len(q2_word_set))
train_df['shared_words'] = train_df.apply(shared_words, axis=1 )<define_variables>
data_all['LotFrontage'] = data_all.groupby(['Neighborhood'])['LotFrontage'].apply(lambda x: x.fillna(x.median()))
data_all.at[2576, 'GarageYrBlt'] = 0
data_all['GarageYrBlt'] = data_all.groupby(['Neighborhood'])['GarageYrBlt'].apply(lambda x: x.fillna(x.median()))
for col in ['MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','BsmtFullBath','BsmtHalfBath','GarageCars',
'GarageArea']:
data_all[col] = data_all[col].fillna(0 )
BERT_VERSION = 'bert-base-uncased'
POOLED_OUTPUT_DIM = 768<load_pretrained>
missing_qual =(data_all[quality].isnull().sum() /data_all[quality].isnull().count() ).sort_values(ascending=False)
 missing_qual = missing_qual[missing_qual > 0] * 100
print("There are {} qualitative features with missing values :".format(missing_qual.shape[0]))
missing_qual = pd.DataFrame({'Percent' :missing_qual})
 missing_qual.head(10 )
tokenizer = BertTokenizer.from_pretrained(BERT_VERSION )<split>
data_all['MSZoning'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['MSZoning'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Utilities'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Utilities'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Exterior1st'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Exterior1st'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Exterior2nd'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Exterior2nd'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['MasVnrType'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['MasVnrType'].apply(lambda x: x.fillna(x.value_counts().index[0]))
for col in ['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','FireplaceQu','GarageType','GarageFinish',
'GarageQual','GarageCond','Alley','Fence','PoolQC','MiscFeature']:
data_all[col] = data_all[col].fillna('None')
 data_all['Electrical'] = data_all.groupby(['Neighborhood','MSSubClass' ])['Electrical'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['KitchenQual'] = data_all.groupby(['Neighborhood','MSSubClass' ])['KitchenQual'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Functional'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Functional'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['SaleType'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['SaleType'].apply(lambda x: x.fillna(x.value_counts().index[0]))
train_df, val_df = train_test_split(train_df, test_size=0.1)
 train_df = train_df.reset_index(drop=True)
 val_df = val_df.reset_index(drop=True )<string_transform>
data_all.iloc[np.where(data_all.GrLivArea > 4000)]
class BertDataSet:
def __init__(self, first_questions, second_questions, targets, tokenizer):
self.first_questions = first_questions
self.second_questions = second_questions
self.targets = targets
self.tokenizer = tokenizer
self.length = len(first_questions)
 def __len__(self):
return self.length
def __getitem__(self, item):
first_question = str(self.first_questions[item])
 second_question = str(self.second_questions[item])
 first_question = " ".join(first_question.split())
 second_question = " ".join(second_question.split())
 inputs = self.tokenizer.encode_plus(
 first_question,
second_question,
add_special_tokens=True,
padding='max_length',
max_length=2 * MAX_LEN + 3,
truncation=True 
)
 return {
"ids": torch.tensor(inputs["input_ids"], dtype=torch.long),
"mask": torch.tensor(inputs["attention_mask"], dtype=torch.long),
"token_type_ids": torch.tensor(inputs["token_type_ids"], dtype=torch.long),
"targets": torch.tensor(int(self.targets[item]), dtype=torch.long)if self.targets is not None else 0
} 
<load_pretrained>
data_all = data_all.drop(data_all[data_all['Id'] == 524].index)
 data_all = data_all.drop(data_all[data_all['Id'] == 1299].index)
 train_len = train_len - 2
data_all.loc[2549,('GrLivArea')] = data_all['GrLivArea'].median()
data_all.loc[2549,('LotArea')] = data_all['LotArea'].median()
def get_data_loader(df, targets, batch_size, shuffle, tokenizer):
dataset = BertDataSet(
 first_questions=df["question1"].values,
second_questions=df["question2"].values,
targets=targets,
tokenizer=tokenizer 
)
 data_loader = torch.utils.data.DataLoader(
 dataset,
batch_size = batch_size,
shuffle=shuffle 
)
 return data_loader<define_variables>
data_all['KitchenQual'].replace(['Ex','Gd','TA','Fa'],[4,3,2,1],inplace=True)
 data_all['FireplaceQu'].replace(['Ex','Gd','TA','Fa','Po', 'None'],[6,5,4,3,2,1],inplace=True)
 data_all['GarageQual'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['GarageCond'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['PoolQC'].replace(['Ex','Gd','TA','Fa','None'],[5,4,3,2,1],inplace=True)
 data_all['ExterQual'].replace(['Ex','Gd','TA','Fa'],[4,3,2,1],inplace=True)
 data_all['ExterCond'].replace(['Ex','Gd','TA','Fa','Po'],[5,4,3,2,1],inplace=True)
 data_all['BsmtQual'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['BsmtCond'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['BsmtExposure'].replace(['Gd','Av','Mn','No','None'],[5,4,3,2,1],inplace=True)
 data_all['HeatingQC'].replace(['Ex','Gd','TA','Fa','Po'],[5,4,3,2,1],inplace=True)
 data_all['MSSubClass'] = data_all['MSSubClass'].astype(str)
 data_all['YrSold'] = data_all['YrSold'].astype(str)
 data_all['MoSold'] = data_all['MoSold'].astype(str )
BS = 128<load_pretrained>
data_all['GarageScale'] = data_all['GarageCars'] * data_all['GarageArea']
data_all['GarageOrdinal'] = data_all['GarageQual'] + data_all['GarageCond']
data_all['AllPorch'] = data_all['OpenPorchSF'] + data_all['EnclosedPorch'] + data_all['3SsnPorch'] + data_all['ScreenPorch']
data_all['ExterOrdinal'] = data_all['ExterQual'] + data_all['ExterCond']
data_all['KitchenCombined'] = data_all['KitchenQual'] * data_all['KitchenAbvGr']
data_all['FireplaceCombined'] = data_all['FireplaceQu'] * data_all['Fireplaces']
data_all['BsmtOrdinal'] = data_all['BsmtQual'] + data_all['BsmtCond']
data_all['BsmtFinishedAll'] = data_all['BsmtFinSF1'] + data_all['BsmtFinSF2']
data_all['AllFlrSF'] = data_all['1stFlrSF'] + data_all['2ndFlrSF']
data_all['OverallCombined'] = data_all['OverallQual'] + data_all['OverallCond']
data_all['TotalFullBath'] = data_all['BsmtFullBath'] + + data_all["FullBath"]
data_all['TotalHalfBath'] = data_all["HalfBath"] + data_all['BsmtHalfBath']
data_all['TotalSF'] = data_all['AllFlrSF'] + data_all['TotalBsmtSF']
data_all['YrBltAndRemod'] = data_all["YearRemodAdd"] + data_all['YearBuilt']
train_data_loader = get_data_loader(
 df=train_df,
targets=train_df["is_duplicate"].values,
batch_size=BS,
shuffle=True,
tokenizer=tokenizer 
)
 val_data_loader = get_data_loader(
 df=val_df,
targets=val_df["is_duplicate"].values,
batch_size=4 * BS,
shuffle=True,
tokenizer=tokenizer
)<choose_model_class>
cat_features = [f for f in data_all.columns if data_all.dtypes[f] == 'object']
categorical = []
for i in data_all.columns:
if i in cat_features:
categorical.append(i)
 data_cat = data_all[categorical]
data_cat = pd.get_dummies(data_cat )
class BertModel(nn.Module):
def __init__(self, bert_path):
super(BertModel, self ).__init__()
self.bert_path = bert_path
self.bert = transformers.BertModel.from_pretrained(self.bert_path)
 self.dropout = nn.Dropout(0.3)
 self.out = nn.Linear(POOLED_OUTPUT_DIM, 1)
 def forward(self, ids, mask, token_type_ids):
_, pooled = self.bert(ids, attention_mask=mask,token_type_ids=token_type_ids)
 pooled = self.dropout(pooled)
 return self.out(pooled)
 model = BertModel(BERT_VERSION ).to(device )<compute_test_metric>
overfit = []
for i in data_cat.columns:
counts = data_cat[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(data_cat)* 100 > 99.95:
overfit.append(i)
 data_cat = data_cat.drop(overfit, axis=1)
 print("There are {} qualitative features with zeros > 99.95 :".format(len(overfit)) )
def loss_fn(outputs, targets):
outputs = torch.squeeze(outputs)
 return nn.BCELoss()(nn.Sigmoid()(outputs), targets )<categorify>
numerical = numeric_cols.values.tolist()
numerical.remove('SalePrice')
 data_num = data_all[numerical].astype(float)
 skewness = data_num.apply(lambda x: skew(x)).sort_values(ascending=False)
 skewness = skewness[abs(skewness)> 0.5]
print("There are {} numerical features with absolute Skew > 0.5 :".format(skewness.shape[0]))
skewness = pd.DataFrame({'Skew' :skewness})
 skewness.head(10 )
def calculate_perplexity(data_loader, model, device):
model.eval()
with torch.no_grad() :
total_loss = 0
for batch in data_loader:
ids = batch["ids"].to(device, dtype=torch.long)
 mask = batch["mask"].to(device, dtype=torch.long)
 token_type_ids = batch["token_type_ids"].to(device, dtype=torch.long)
 targets = batch["targets"].to(device, dtype=torch.float)
 outputs = model(ids=ids, mask=mask, token_type_ids=token_type_ids)
 total_loss += loss_fn(outputs, targets ).item()
model.train()
return np.exp(total_loss / len(data_loader))<train_model>
for i in skewness.index.tolist() :
data_num[i] = boxcox1p(data_num[i], boxcox_normmax(data_num[i] + 1))
def train_loop(epochs, train_data_loader, val_data_loader, model, optimizer, device, scheduler=None):
it = 1
total_loss = 0
curr_perplexity = None
perplexity = None
model.train()
for epoch in range(epochs):
print('Epoch: ', epoch + 1)
 for batch in train_data_loader:
ids = batch["ids"].to(device, dtype=torch.long)
 mask = batch["mask"].to(device, dtype=torch.long)
 token_type_ids = batch["token_type_ids"].to(device, dtype=torch.long)
 targets = batch["targets"].to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(ids, mask=mask, token_type_ids=token_type_ids)
 loss = loss_fn(outputs, targets)
 total_loss += loss.item()
loss.backward()
optimizer.step()
if it % 100 == 0:
if it % 500 == 0:
curr_perplexity = calculate_perplexity(val_data_loader, model, device)
 if scheduler is not None:
scheduler.step()
if not perplexity or curr_perplexity < perplexity:
torch.save(model.state_dict() , 'saved_model')
 perplexity = curr_perplexity
print('| Iter', it, '| Avg Train Loss', total_loss / 100, '| Dev Perplexity', curr_perplexity)
 total_loss = 0
it += 1 
<train_model>
skewness = data_num.apply(lambda x: skew(x)).sort_values(ascending=False)
 skewness = skewness[abs(skewness)> 0.5]
neg_skew = skewness[skewness < 0]
data_num.loc[:,('GarageOrdinal')] = data_num['GarageOrdinal']**1.1
data_num.loc[:,('GarageQual')] = data_num['GarageQual']**1.2
data_num.loc[:,('GarageCond')] = data_num['GarageCond']**1.3
data_num.loc[:,('HeatingQC')] = data_num['HeatingQC']**1.6
pos_skew = skewness[skewness > 0]
pos_skew1 = ['Fireplaces','BsmtFullBath','OpenPorchSF','FireplaceCombined','TotalHalfBath',
'WoodDeckSF','2ndFlrSF']
data_num.loc[:,pos_skew1] = np.sqrt(data_num[pos_skew1])
 data_num.loc[:,('HalfBath')] = np.cbrt(data_num['HalfBath'])
 data_num.loc[:,('MasVnrArea')] = np.log2(data_num['MasVnrArea']+1)
 skewness = data_num.apply(lambda x: skew(x)).sort_values(ascending=False)
 skewness = skewness[abs(skewness)> 1.0]
data_num = data_num.drop(skewness.index.tolist() , axis=1)
 print("There are {} numerical features with absolute Skew > 1.0 :".format(skewness.shape[0]))
def run(model, train_df, device, train_data_loader, val_data_loader):
EPOCHS = 1
lr = 3e-5
num_training_steps = int(len(train_data_loader)* EPOCHS)
 optimizer = AdamW(model.parameters() , lr=lr)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_training_steps 
)
 train_loop(EPOCHS, train_data_loader, val_data_loader, model, optimizer, device, scheduler )<train_model>
data_num =(data_num - data_num.mean())/data_num.std()
df_all = pd.concat([data_num, data_cat], axis = 1)
 features =(data_cat.columns ).append(data_num.columns)
 df_train = df_all[:1458]
df_test = df_all[1458:]
target = 'SalePrice'
X = df_train[features]
y = data_all[:1458][target]
X_test = df_test[features]
run(model, train_df, device, train_data_loader, val_data_loader )<load_from_csv>
def rmse_cv(model):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring = "neg_mean_squared_error", cv = 5))
return(rmse )
test_df = pd.read_csv("/kaggle/input/quora-question-pairs/test.csv")
 test_df.head()<split>
ridge = Ridge(alpha = 14)
 cv_ridge = rmse_cv(ridge)
 ridge.fit(X , y)
 print('Ridge CV score min: ' + str(cv_ridge.min())+ ' mean: ' + str(cv_ridge.mean())
 + ' max: ' + str(cv_ridge.max()))
def test(model, test_df, device):
predictions = torch.empty(0 ).to(device, dtype=torch.float)
 test_dataset = BertDataSet(
 first_questions=test_df["question1"].values,
second_questions=test_df["question2"].values,
targets=None,
tokenizer=tokenizer 
)
 test_data_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=512 
)
 with torch.no_grad() :
model.eval()
for batch in tqdm(test_data_loader):
ids = batch["ids"]
mask = batch["mask"]
token_type_ids = batch["token_type_ids"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 outputs = model(ids=ids, mask=mask, token_type_ids=token_type_ids)
 predictions = torch.cat(( predictions, nn.Sigmoid()(outputs)))
 return predictions.cpu().numpy().squeeze()
predictions = test(model, test_df, device)
 len(predictions )<prepare_output>
lasso = Lasso(alpha =.00066)
 cv_lasso = rmse_cv(lasso)
 lasso.fit(X, y)
 print('Lasso CV score min: ' + str(cv_lasso.min())+ ' mean: ' + str(cv_lasso.mean())
 + ' max: ' + str(cv_lasso.max()))
test_df['is_duplicate'] = predictions<save_to_csv>
gbrt = GradientBoostingRegressor(n_estimators=1000, learning_rate=.03, max_depth=3, max_features=.04, min_samples_split=4,
min_samples_leaf=3, loss='huber', subsample=1.0, random_state=0)
 cv_gbrt = rmse_cv(gbrt)
 gbrt.fit(X, y)
 print('GradientBoosting CV score min: ' + str(cv_gbrt.min())+ ' mean: ' + str(cv_gbrt.mean())
 + ' max: ' + str(cv_gbrt.max()))
test_df[['test_id', 'is_duplicate']].to_csv('submission.csv', index=False )<categorify>
def blend_models(X):
return(( ridge.predict(X)) +(gbrt.predict(X)))/2
ridge_preds = np.expm1(ridge.predict(X_test))
gbrt_preds = np.expm1(gbrt.predict(X_test))
ridge_gbrt_preds = np.expm1(blend_models(X_test))
hp_ridge_gbrt = pd.DataFrame({'Id':test.Id, 'SalePrice':ridge_gbrt_preds})
 hp_ridge_gbrt.to_csv('solution.csv', index = False )
def eval(model, tokenizer, first_question, second_question, device):
inputs = tokenizer.encode_plus(
 first_question,
second_question,
add_special_tokens=True, 
)
 ids = torch.tensor([inputs["input_ids"]], dtype=torch.long ).to(device, dtype=torch.long)
 mask = torch.tensor([inputs["attention_mask"]], dtype=torch.long ).to(device, dtype=torch.long)
 token_type_ids = torch.tensor([inputs["token_type_ids"]], dtype=torch.long ).to(device, dtype=torch.long)
 with torch.no_grad() :
model.eval()
output = model(ids=ids, mask=mask, token_type_ids=token_type_ids)
 prob = nn.Sigmoid()(output ).item()
print("questions [{}] and [{}] are {} with score {}".format(first_question, second_question, 'similar' if prob > 0.5 else 'not similar', prob))<import_modules>
plt.style.use('classic')
 %matplotlib inline
sb.set()
warnings.filterwarnings('ignore' )
import pandas as pd
import numpy as np
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding,Activation,Flatten,merge,TimeDistributed,CuDNNGRU,Bidirectional
from keras.layers.merge import concatenate
from keras.models import Model
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.layers import concatenate,subtract,add,maximum,multiply,Layer,Lambda
from keras.backend import backend as K
from tqdm import tqdm 
<load_from_csv>
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train_len = len(train)
 data_all = train.append(test, sort=False, ignore_index=True)
 features = data_all.shape[1]
idUnique = len(set(data_all.Id))
idTotal = data_all.shape[0]
idDupli = idTotal - idUnique
print("There are " + str(idDupli)+ " duplicate IDs for " + str(idTotal)+ " total entries")
 print("There are " + str(features)+ " variables for " + str(idTotal)+ " total entries" )
MAX_SEQUENCE_LENGTH = 40
MAX_NB_WORDS = 4000000
nrows = 10000000
EMBEDDING_DIM = 300
path = '.. /input/quora-question-pairs/'
train = pd.read_csv(path+"train.csv",nrows=nrows ).astype(str)
 test = pd.read_csv(path+"test.csv",nrows=nrows ).astype(str )<data_type_conversions>
quantity = [f for f in data_all.columns if data_all.dtypes[f] != 'object']
quantity.remove('SalePrice')
 quantity.remove('Id')
 quality = [f for f in data_all.columns if data_all.dtypes[f] == 'object']
corpus = []
feats = ['question1','question2']
for f in feats:
train[f] = train[f].astype(str)
 test[f] = test[f].astype(str)
 corpus+=train[f].values.tolist()
tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
 tokenizer.fit_on_texts(corpus)
 X_q1 = tokenizer.texts_to_sequences(train['question1'])
 X_q2 = tokenizer.texts_to_sequences(train['question2'])
 X_test_q1 = tokenizer.texts_to_sequences(test['question1'])
 X_test_q2 = tokenizer.texts_to_sequences(test['question2'])
 X_q1 = pad_sequences(X_q1, maxlen=MAX_SEQUENCE_LENGTH)
 X_q2 = pad_sequences(X_q2, maxlen=MAX_SEQUENCE_LENGTH)
 X_test_q1 = pad_sequences(X_test_q1, maxlen=MAX_SEQUENCE_LENGTH)
 X_test_q2 = pad_sequences(X_test_q2, maxlen=MAX_SEQUENCE_LENGTH)
 y = train['is_duplicate'].values
word_index = tokenizer.word_index
print('Found %s unique tokens' % len(word_index))
nb_words = min(MAX_NB_WORDS, len(word_index)) +1 
<feature_engineering>
missing_quant =(data_all[quantity].isnull().sum() /data_all[quantity].isnull().count() ).sort_values(ascending=False)
 missing_quant = missing_quant[missing_quant > 0] * 100
print("There are {} quantitative features with missing values :".format(missing_quant.shape[0]))
missing_quant = pd.DataFrame({'Percent' :missing_quant})
 missing_quant.head()
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
glove_path = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
embedding_matrix,unknown_words = build_matrix(word_index,glove_path )<import_modules>
np.where(pd.isnull(data_all.GarageArea))
from sklearn.metrics import log_loss<split>
data_all['LotFrontage'] = data_all.groupby(['Neighborhood'])['LotFrontage'].apply(lambda x: x.fillna(x.median()))
data_all.at[2576, 'GarageYrBlt'] = 0
data_all['GarageYrBlt'] = data_all.groupby(['Neighborhood'])['GarageYrBlt'].apply(lambda x: x.fillna(x.median()))
for col in ['MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','BsmtFullBath','BsmtHalfBath','GarageCars',
'GarageArea']:
data_all[col] = data_all[col].fillna(0 )
X_train_q1,X_val_q1,X_train_q2,X_val_q2,y_train,y_val = train_test_split(X_q1,X_q2,y,train_size=0.8,random_state=1024)
 print(X_train_q1.shape,X_val_q1.shape)
 X_train = [X_train_q1,X_train_q2]
X_val = [X_val_q1,X_val_q2]
X_test = [X_test_q1,X_test_q2] 
<choose_model_class>
missing_qual =(data_all[quality].isnull().sum() /data_all[quality].isnull().count() ).sort_values(ascending=False)
 missing_qual = missing_qual[missing_qual > 0] * 100
print("There are {} qualitative features with missing values :".format(missing_qual.shape[0]))
missing_qual = pd.DataFrame({'Percent' :missing_qual})
 missing_qual.head(10 )
def get_model_siamese(nb_words=50000,MAX_SEQUENCE_LENGTH=200,EMBEDDING_DIM=200,act='relu',embedding_matrix=None):
input_q1 = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
 input_q2 = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
 if embedding_matrix is not None:
weights=[embedding_matrix],
embedding_layer = Embedding(nb_words,
EMBEDDING_DIM,
input_length=MAX_SEQUENCE_LENGTH,
weights=[embedding_matrix],
trainable=False)
 else:
embedding_layer = Embedding(nb_words,
EMBEDDING_DIM,
input_length=MAX_SEQUENCE_LENGTH,
trainable=True)
 embedded_sequences_q1 = embedding_layer(input_q1)
 embedded_sequences_q2 = embedding_layer(input_q2)
 bilstm_layer = Bidirectional(CuDNNGRU(64,return_sequences=False))
x1 = bilstm_layer(embedded_sequences_q1)
 x2 = bilstm_layer(embedded_sequences_q2)
 diff = subtract([x1,x2])
 summation = add([x1,x2])
 x = concatenate([x1,x2,diff,summation],1)
 x = Dense(64,activation=act )(x)
 preds = Dense(1,activation='sigmoid' )(x)
 model = Model(inputs=[input_q1,input_q2],outputs=preds)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 print(model.summary())
 return model<train_model>
data_all['MSZoning'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['MSZoning'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Utilities'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Utilities'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Exterior1st'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Exterior1st'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Exterior2nd'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Exterior2nd'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['MasVnrType'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['MasVnrType'].apply(lambda x: x.fillna(x.value_counts().index[0]))
for col in ['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','FireplaceQu','GarageType','GarageFinish',
'GarageQual','GarageCond','Alley','Fence','PoolQC','MiscFeature']:
data_all[col] = data_all[col].fillna('None')
 data_all['Electrical'] = data_all.groupby(['Neighborhood','MSSubClass' ])['Electrical'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['KitchenQual'] = data_all.groupby(['Neighborhood','MSSubClass' ])['KitchenQual'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['Functional'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['Functional'].apply(lambda x: x.fillna(x.value_counts().index[0]))
data_all['SaleType'] = data_all.groupby(['Neighborhood', 'MSSubClass'])['SaleType'].apply(lambda x: x.fillna(x.value_counts().index[0]))
early_stopping =EarlyStopping(monitor='val_loss', patience=5)
 bst_model_path = 'best_model.h5'
model_checkpoint = ModelCheckpoint(bst_model_path, save_best_only=True, save_weights_only=True)
 model = get_model_siamese(nb_words=nb_words,MAX_SEQUENCE_LENGTH=MAX_SEQUENCE_LENGTH,EMBEDDING_DIM=EMBEDDING_DIM,act='relu',embedding_matrix=embedding_matrix)
 model.fit(X_train,y_train,epochs=250,validation_data=(X_val, y_val),batch_size=128,shuffle=True,verbose=1,callbacks=[early_stopping, model_checkpoint])
 model.load_weights(bst_model_path)
<save_to_csv>
data_all.iloc[np.where(data_all.GrLivArea > 4000)]
y_pred_test = model.predict(X_test)
 submission = pd.DataFrame()
submission['test_id'] = test['test_id']
submission['is_duplicate'] = y_pred_test
submission.to_csv('submission_siamese.csv',index=False)
<set_options>
data_all = data_all.drop(data_all[data_all['Id'] == 524].index)
 data_all = data_all.drop(data_all[data_all['Id'] == 1299].index)
 train_len = train_len - 2
data_all.loc[2549,('GrLivArea')] = data_all['GrLivArea'].median()
data_all.loc[2549,('LotArea')] = data_all['LotArea'].median()
warnings.filterwarnings("ignore")
<load_from_csv>
data_all['KitchenQual'].replace(['Ex','Gd','TA','Fa'],[4,3,2,1],inplace=True)
 data_all['FireplaceQu'].replace(['Ex','Gd','TA','Fa','Po', 'None'],[6,5,4,3,2,1],inplace=True)
 data_all['GarageQual'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['GarageCond'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['PoolQC'].replace(['Ex','Gd','TA','Fa','None'],[5,4,3,2,1],inplace=True)
 data_all['ExterQual'].replace(['Ex','Gd','TA','Fa'],[4,3,2,1],inplace=True)
 data_all['ExterCond'].replace(['Ex','Gd','TA','Fa','Po'],[5,4,3,2,1],inplace=True)
 data_all['BsmtQual'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['BsmtCond'].replace(['Ex','Gd','TA','Fa','Po','None'],[6,5,4,3,2,1],inplace=True)
 data_all['BsmtExposure'].replace(['Gd','Av','Mn','No','None'],[5,4,3,2,1],inplace=True)
 data_all['HeatingQC'].replace(['Ex','Gd','TA','Fa','Po'],[5,4,3,2,1],inplace=True)
 data_all['MSSubClass'] = data_all['MSSubClass'].astype(str)
 data_all['YrSold'] = data_all['YrSold'].astype(str)
 data_all['MoSold'] = data_all['MoSold'].astype(str )
train = pd.read_csv(".. /input/quora-question-pairs-feature-extraction-2/train.csv")
 test = pd.read_csv(".. /input/quora-question-pairs-feature-extraction-2/test.csv")
 trainlabel = pd.read_csv(".. /input/quora-question-pairs-feature-extraction-2/trainlabel.csv" )<prepare_x_and_y>
data_all['GarageScale'] = data_all['GarageCars'] * data_all['GarageArea']
data_all['GarageOrdinal'] = data_all['GarageQual'] + data_all['GarageCond']
data_all['AllPorch'] = data_all['OpenPorchSF'] + data_all['EnclosedPorch'] + data_all['3SsnPorch'] + data_all['ScreenPorch']
data_all['ExterOrdinal'] = data_all['ExterQual'] + data_all['ExterCond']
data_all['KitchenCombined'] = data_all['KitchenQual'] * data_all['KitchenAbvGr']
data_all['FireplaceCombined'] = data_all['FireplaceQu'] * data_all['Fireplaces']
data_all['BsmtOrdinal'] = data_all['BsmtQual'] + data_all['BsmtCond']
data_all['BsmtFinishedAll'] = data_all['BsmtFinSF1'] + data_all['BsmtFinSF2']
data_all['AllFlrSF'] = data_all['1stFlrSF'] + data_all['2ndFlrSF']
data_all['OverallCombined'] = data_all['OverallQual'] + data_all['OverallCond']
data_all['TotalFullBath'] = data_all['BsmtFullBath'] + + data_all["FullBath"]
data_all['TotalHalfBath'] = data_all["HalfBath"] + data_all['BsmtHalfBath']
data_all['TotalSF'] = data_all['AllFlrSF'] + data_all['TotalBsmtSF']
data_all['YrBltAndRemod'] = data_all["YearRemodAdd"] + data_all['YearBuilt']
dtrain = xgb.DMatrix(train, label = trainlabel )<feature_engineering>
cat_features = [f for f in data_all.columns if data_all.dtypes[f] == 'object']
categorical = []
for i in data_all.columns:
if i in cat_features:
categorical.append(i)
 data_cat = data_all[categorical]
data_cat = pd.get_dummies(data_cat )
w0 = average *(1 - p)/(( 1 - average)* p)
 print(w0 )<compute_test_metric>
overfit = []
for i in data_cat.columns:
counts = data_cat[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(data_cat)* 100 > 99.95:
overfit.append(i)
 data_cat = data_cat.drop(overfit, axis=1)
 print("There are {} qualitative features with zeros > 99.95 :".format(len(overfit)) )
w1 = average / p
w2 =(1 - average)/(1 - p)
 print(w1, w2 )<compute_test_metric>
numerical = numeric_cols.values.tolist()
numerical.remove('SalePrice')
 data_num = data_all[numerical].astype(float)
 skewness = data_num.apply(lambda x: skew(x)).sort_values(ascending=False)
 skewness = skewness[abs(skewness)> 0.5]
print("There are {} numerical features with absolute Skew > 0.5 :".format(skewness.shape[0]))
skewness = pd.DataFrame({'Skew' :skewness})
 skewness.head(10 )
def weighted_log_loss(preds, dtrain):
label = dtrain.get_label()
return "weighted_logloss", -np.mean(w1 * label * np.log(preds)+ w2 *(1 - label)* np.log(1 - preds))<train_on_grid>
for i in skewness.index.tolist() :
data_num[i] = boxcox1p(data_num[i], boxcox_normmax(data_num[i] + 1))
params = {}
params["objective"] = "binary:logistic"
params["eval_metric"] = "logloss"
params["eta"] = 0.1
params["max_depth"] = 6
params["min_child_weight"] = 1
params["gamma"] = 0
params["subsample"] = 0.8
params["colsample_bytree"] = 0.9
params["scale_pos_weight"] = 0.3632
params["tree_method"] = "gpu_hist"
params['max_bin'] = 256
model1 = xgb.cv(params, dtrain, num_boost_round = 2000, nfold = 10,
feval = weighted_log_loss, early_stopping_rounds = 200,
verbose_eval = 50 )<find_best_params>
skewness = data_num.apply(lambda x: skew(x)).sort_values(ascending=False)
 skewness = skewness[abs(skewness)> 0.5]
neg_skew = skewness[skewness < 0]
data_num.loc[:,('GarageOrdinal')] = data_num['GarageOrdinal']**1.1
data_num.loc[:,('GarageQual')] = data_num['GarageQual']**1.2
data_num.loc[:,('GarageCond')] = data_num['GarageCond']**1.3
data_num.loc[:,('HeatingQC')] = data_num['HeatingQC']**1.6
pos_skew = skewness[skewness > 0]
pos_skew1 = ['Fireplaces','BsmtFullBath','OpenPorchSF','FireplaceCombined','TotalHalfBath',
'WoodDeckSF','2ndFlrSF']
data_num.loc[:,pos_skew1] = np.sqrt(data_num[pos_skew1])
 data_num.loc[:,('HalfBath')] = np.cbrt(data_num['HalfBath'])
 data_num.loc[:,('MasVnrArea')] = np.log2(data_num['MasVnrArea']+1)
 skewness = data_num.apply(lambda x: skew(x)).sort_values(ascending=False)
 skewness = skewness[abs(skewness)> 1.0]
data_num = data_num.drop(skewness.index.tolist() , axis=1)
 print("There are {} numerical features with absolute Skew > 1.0 :".format(skewness.shape[0]))
fix_params = {}
fix_params["objective"] = "binary:logistic"
fix_params["eval_metric"] = "logloss"
fix_params["eta"] = 0.1
fix_params["gamma"] = 0
fix_params["subsample"] = 0.8
fix_params["colsample_bytree"] = 0.9
fix_params["scale_pos_weight"] = 0.3632
fix_params["tree_method"] = "gpu_hist"
fix_params["max_bin"] = 256
evaluation_list = []
for depth in [5, 6]:
for child_weight in [1, 2.5, 4]:
params = {**fix_params, **{"max_depth": depth, "min_child_weight": child_weight}}
evaluation = xgb.cv(params, dtrain, num_boost_round = 650, nfold = 6,
feval = weighted_log_loss, early_stopping_rounds = 100)
 evaluation_list.append(evaluation)
 for depth in [7, 8]:
for child_weight in [4, 5, 6]:
params = {**fix_params, **{"max_depth": depth, "min_child_weight": child_weight}}
evaluation = xgb.cv(params, dtrain, num_boost_round = 650, nfold = 6,
feval = weighted_log_loss, early_stopping_rounds = 100)
 evaluation_list.append(evaluation)
 evaluation_panel = pd.DataFrame()
for evaluation in evaluation_list:
evaluation_panel = pd.concat([evaluation_panel, evaluation.iloc[-1, :]], axis = 1)
 evaluation_panel<find_best_params>
data_num =(data_num - data_num.mean())/data_num.std()
df_all = pd.concat([data_num, data_cat], axis = 1)
 features =(data_cat.columns ).append(data_num.columns)
 df_train = df_all[:1458]
df_test = df_all[1458:]
target = 'SalePrice'
X = df_train[features]
y = data_all[:1458][target]
X_test = df_test[features]
fix_params = {}
fix_params["objective"] = "binary:logistic"
fix_params["eval_metric"] = "logloss"
fix_params["eta"] = 0.1
fix_params["gamma"] = 0
fix_params["subsample"] = 0.8
fix_params["colsample_bytree"] = 0.9
fix_params["scale_pos_weight"] = 0.3632
fix_params["tree_method"] = "gpu_hist"
fix_params["max_bin"] = 256
evaluation_list = []
for depth in [5, 6, 7]:
for child_weight in [3, 3.5, 4, 4.5]:
params = {**fix_params, **{"max_depth": depth, "min_child_weight": child_weight}}
evaluation = xgb.cv(params, dtrain, num_boost_round = 650, nfold = 6,
feval = weighted_log_loss, early_stopping_rounds = 100)
 evaluation_list.append(evaluation)
 evaluation_panel = pd.DataFrame()
for evaluation in evaluation_list:
evaluation_panel = pd.concat([evaluation_panel, evaluation.iloc[-1, :]], axis = 1)
 evaluation_panel<find_best_params>
def rmse_cv(model):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring = "neg_mean_squared_error", cv = 5))
return(rmse )
fix_params = {}
fix_params["objective"] = "binary:logistic"
fix_params["eval_metric"] = "logloss"
fix_params["eta"] = 0.1
fix_params["gamma"] = 0
fix_params["subsample"] = 0.8
fix_params["colsample_bytree"] = 0.9
fix_params["scale_pos_weight"] = 0.3632
fix_params["tree_method"] = "gpu_hist"
fix_params["max_depth"] = 6
fix_params["min_child_weight"] = 4
evaluation_list = []
for bin in [200, 230, 256, 280]:
params = {**fix_params, **{"max_bin": bin}}
evaluation = xgb.cv(params, dtrain, num_boost_round = 650, nfold = 6,
feval = weighted_log_loss, early_stopping_rounds = 100)
 evaluation_list.append(evaluation)
 evaluation_panel = pd.DataFrame()
for evaluation in evaluation_list:
evaluation_panel = pd.concat([evaluation_panel, evaluation.iloc[-1, :]], axis = 1)
 evaluation_panel<find_best_params>
ridge = Ridge(alpha = 14)
 cv_ridge = rmse_cv(ridge)
 ridge.fit(X , y)
 print('Ridge CV score min: ' + str(cv_ridge.min())+ ' mean: ' + str(cv_ridge.mean())
 + ' max: ' + str(cv_ridge.max()))
fix_params = {}
fix_params["objective"] = "binary:logistic"
fix_params["eval_metric"] = "logloss"
fix_params["eta"] = 0.08
fix_params["gamma"] = 0
fix_params["subsample"] = 0.8
fix_params["colsample_bytree"] = 0.9
fix_params["scale_pos_weight"] = 0.3632
fix_params["tree_method"] = "gpu_hist"
fix_params["max_depth"] = 6
fix_params["min_child_weight"] = 3.5
evaluation_list = []
for bin in [220, 240, 270]:
params = {**fix_params, **{"max_bin": bin}}
evaluation = xgb.cv(params, dtrain, num_boost_round = 650, nfold = 6,
feval = weighted_log_loss, early_stopping_rounds = 100)
 evaluation_list.append(evaluation)
 evaluation_panel = pd.DataFrame()
for evaluation in evaluation_list:
evaluation_panel = pd.concat([evaluation_panel, evaluation.iloc[-1, :]], axis = 1)
 evaluation_panel<init_hyperparams>
lasso = Lasso(alpha =.00066)
 cv_lasso = rmse_cv(lasso)
 lasso.fit(X, y)
 print('Lasso CV score min: ' + str(cv_lasso.min())+ ' mean: ' + str(cv_lasso.mean())
 + ' max: ' + str(cv_lasso.max()))
fix_params = {}
fix_params["objective"] = "binary:logistic"
fix_params["eval_metric"] = "logloss"
fix_params["eta"] = 0.1
fix_params["gamma"] = 0
fix_params["scale_pos_weight"] = 0.3632
fix_params["tree_method"] = "gpu_hist"
fix_params["max_depth"] = 6
fix_params["min_child_weight"] = 4
fix_params["max_bin"] = 256
evaluation_list = []
for row in [0.7, 0.8, 0.9]:
for col in [0.7, 0.8, 0.9]:
params = {**fix_params, **{"subsample": row, "colsample_bytree": col}}
evaluation = xgb.cv(params, dtrain, num_boost_round = 650, nfold = 6,
feval = weighted_log_loss, early_stopping_rounds = 100)
 evaluation_list.append(evaluation)
 evaluation_panel = pd.DataFrame()
for evaluation in evaluation_list:
evaluation_panel = pd.concat([evaluation_panel, evaluation.iloc[-1, :]], axis = 1)
 evaluation_panel<find_best_params>
gbrt = GradientBoostingRegressor(n_estimators=1000, learning_rate=.03, max_depth=3, max_features=.04, min_samples_split=4,
min_samples_leaf=3, loss='huber', subsample=1.0, random_state=0)
 cv_gbrt = rmse_cv(gbrt)
 gbrt.fit(X, y)
 print('GradientBoosting CV score min: ' + str(cv_gbrt.min())+ ' mean: ' + str(cv_gbrt.mean())
 + ' max: ' + str(cv_gbrt.max()))
<init_hyperparams><EOS> 
def blend_models(X):
return(( ridge.predict(X)) +(gbrt.predict(X)))/2
ridge_preds = np.expm1(ridge.predict(X_test))
gbrt_preds = np.expm1(gbrt.predict(X_test))
ridge_gbrt_preds = np.expm1(blend_models(X_test))
hp_ridge_gbrt = pd.DataFrame({'Id':test.Id, 'SalePrice':ridge_gbrt_preds})
 hp_ridge_gbrt.to_csv('solution.csv', index = False )