Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<count_values>
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.width', 1000)
 if not sys.warnoptions:
warnings.simplefilter("ignore" )
train.Sentiment.value_counts()<install_modules>
desc = {
'SalePrice': "The property's sale price in dollars.This is the target variable to be predicted",
'MSSubClass': "The building class",
'MSZoning': "The general zoning classification",
'LotFrontage': "Linear feet of street connected to property",
'LotArea': "Lot size in square feet",
'Street': "Type of road access",
'Alley': "Type of alley access",
'LotShape': "General shape of property",
'LandContour': "Flatness of the property",
'Utilities': "Type of utilities available",
'LotConfig': "Lot configuration",
'LandSlope': "Slope of property",
'Neighborhood': "Physical locations within Ames city limits",
'Condition1': "Proximity to main road or railroad",
'Condition2': "Proximity to main road or railroad(if a second is present)",
'BldgType': "Type of dwelling",
'HouseStyle': "Style of dwelling",
'OverallQual': "Overall material and finish quality",
'OverallCond': "Overall condition rating",
'YearBuilt': "Original construction date",
'YearRemodAdd': "Remodel date",
'RoofStyle': "Type of roof",
'RoofMatl': "Roof material",
'Exterior1st': "Exterior covering on house",
'Exterior2nd': "Exterior covering on house(if more than one material)",
'MasVnrType': "Masonry veneer type",
'MasVnrArea': "Masonry veneer area in square feet",
'ExterQual': "Exterior material quality",
'ExterCond': "Present condition of the material on the exterior",
'Foundation': "Type of foundation",
'BsmtQual': "Height of the basement",
'BsmtCond': "General condition of the basement",
'BsmtExposure': "Walkout or garden level basement walls",
'BsmtFinType1': "Quality of basement finished area",
'BsmtFinSF1': "Type 1 finished square feet",
'BsmtFinType2': "Quality of second finished area(if present)",
'BsmtFinSF2': "Type 2 finished square feet",
'BsmtUnfSF': "Unfinished square feet of basement area",
'TotalBsmtSF': "Total square feet of basement area",
'Heating': "Type of heating",
'HeatingQC': "Heating quality and condition",
'CentralAir': "Central air conditioning",
'Electrical':" Electrical system",
'1stFlrSF': "First Floor square feet",
'2ndFlrSF': "Second floor square feet",
'LowQualFinSF': "Low quality finished square feet(all floors)",
'GrLivArea': "Above grade(ground)living area square feet",
'BsmtFullBath': "Basement full bathrooms",
'BsmtHalfBath': "Basement half bathrooms",
'FullBath': "Full bathrooms above grade",
'HalfBath': "Half baths above grade",
'Bedroom': "Number of bedrooms above basement level",
'Kitchen': "Number of kitchens",
'KitchenQual': "Kitchen quality",
'TotRmsAbvGrd': "Total rooms above grade(does not include bathrooms)",
'Functional': "Home functionality rating",
'Fireplaces': "Number of fireplaces",
'FireplaceQu': "Fireplace quality",
'GarageType': "Garage location",
'GarageYrBlt': "Year garage was built",
'GarageFinish': "Interior finish of the garage",
'GarageCars': "Size of garage in car capacity",
'GarageArea': "Size of garage in square feet",
'GarageQual': "Garage quality",
'GarageCond': "Garage condition",
'PavedDrive': "Paved driveway",
'WoodDeckSF': "Wood deck area in square feet",
'OpenPorchSF': "Open porch area in square feet",
'EnclosedPorch': "Enclosed porch area in square feet",
'3SsnPorch': "Three season porch area in square feet",
'ScreenPorch': "Screen porch area in square feet",
'PoolArea': "Pool area in square feet",
'PoolQC': "Pool quality",
'Fence': "Fence quality",
'MiscFeature': "Miscellaneous feature not covered in other categories",
'MiscVal': "Value of miscellaneous feature",
'MoSold': "Month Sold",
'YrSold': "Year Sold",
'SaleType': "Type of sale",
'SaleCondition': "Condition of sale"
}
!pip install pytorch-pretrained-bert pytorch-nlp<set_options>
json.dump(desc, open("desc.json",'w'))
with open("desc.json", "r")as read_file:
d= json.load(read_file )
% matplotlib inline<set_options>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 n_gpu = torch.cuda.device_count()
torch.cuda.get_device_name(0 )<data_type_conversions>
train_ID = train['Id']
test_ID = test['Id']
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True)
 numerical = [x for x in train.columns if train.dtypes[x] != 'object']
numerical.remove('SalePrice')
 categorical = [x for x in train.columns if train.dtypes[x] == 'object']
y = train.SalePrice.reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 data_overview(features )
train['Sentiment'] = train.Sentiment.astype(int)
 test['Sentiment'] = test.Sentiment.astype(int )<define_variables>
train["SalePrice"] = np.log1p(train["SalePrice"])
 y = train.SalePrice.reset_index(drop=True )
sentences = train.Phrase.values
sentences = ["[CLS] " + sentence + " [SEP]" for sentence in sentences]
labels = train.Sentiment.values<string_transform>
corr_dict = train.corr() ['SalePrice'][(train.corr() ['SalePrice'] > 0.5)&(train.corr() ['SalePrice'] < 1.0)].sort_values(ascending=False ).to_dict()
print("Higher correlations between predictor variables and the response variable ")
 print("-------------------------------------------------------------------------")
 for x in corr_dict.keys() :
print("(" , round(corr_dict[x],3), ")",x, ":" , " -",desc[x] )
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
 tokenized_texts = [tokenizer.tokenize(sent)for sent in sentences]
print("Tokenize the first sentence:")
 print(tokenized_texts[0] )<import_modules>
features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str )
import seaborn as sns<define_variables>
for f in('MasVnrType','Alley','PoolQC', 'Fence', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',):
features[f] = features[f].fillna('None' )
MAX_LEN = 32<string_transform>
features['MiscFeature'] = features['MiscFeature'].fillna('None' )
input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt)for txt in tokenized_texts],
maxlen=MAX_LEN, dtype="long", truncating="post", padding="post" )<categorify>
for f in('Exterior1st', 'Exterior2nd', 'SaleType','Utilities'):
features[f] = features[f].fillna(features[f].mode() [0])
 features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
input_ids = [tokenizer.convert_tokens_to_ids(x)for x in tokenized_texts]<string_transform>
features[features['GarageYrBlt'] == 2207][['YearBuilt','YearRemodAdd','GarageYrBlt']]
input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype="long", truncating="post", padding="post" )<define_variables>
features.GarageYrBlt.iloc[2592] = 2007
attention_masks = []
for seq in input_ids:
seq_mask = [float(i>0)for i in seq]
attention_masks.append(seq_mask )<split>
for f in ['GarageYrBlt', 'GarageArea', 'GarageCars']:
features[f] = features[f].fillna(0 )
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels,
random_state=2018, test_size=0.1)
 train_masks, validation_masks, _, _ = train_test_split(attention_masks, input_ids,
random_state=2018, test_size=0.1 )<data_type_conversions>
for f in ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath']:
features[f] = features[f].fillna(0 )
train_inputs = torch.tensor(train_inputs)
 validation_inputs = torch.tensor(validation_inputs)
 train_labels = torch.tensor(train_labels)
 validation_labels = torch.tensor(validation_labels)
 train_masks = torch.tensor(train_masks)
 validation_masks = torch.tensor(validation_masks )<split>
features['MasVnrArea'] = features['MasVnrArea'].fillna(0 )
batch_size = 32
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)
<load_pretrained>
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].apply(lambda x: x.fillna(x.median()))
model = BertForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=5)
 model.cuda()<init_hyperparams>
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics2 = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics2.append(i)
 skew_features = features[numerics2].apply(lambda x: stats.skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))
param_optimizer = list(model.named_parameters())
 no_decay = ['bias', 'gamma', 'beta']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay_rate': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay_rate': 0.0}
] 
<choose_model_class>
features['HasPool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['Has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['HasGarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['HasBsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['HasFireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['Total_SQR_Footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['Total_Porch_SF'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
 features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1 )
optimizer = BertAdam(optimizer_grouped_parameters,
lr=2e-5,
warmup=.1 )<compute_test_metric>
final_features = pd.get_dummies(features ).reset_index(drop=True)
 print(final_features.shape )
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )<train_model>
X = final_features.iloc[:len(y), :]
X_pred = final_features.iloc[len(X):, :]
X_train = X
X_test = X_pred
y_train = y
print('X', X.shape, 'y', y.shape, 'X_pred', X_pred.shape )
train_loss_set = []
epochs = 4
for _ in trange(epochs, desc="Epoch"):
model.train()
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(train_dataloader):
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
optimizer.zero_grad()
loss = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)
 train_loss_set.append(loss.item())
 loss.backward()
optimizer.step()
tr_loss += loss.item()
nb_tr_examples += b_input_ids.size(0)
 nb_tr_steps += 1
print("Train loss: {}".format(tr_loss/nb_tr_steps))
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in validation_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
logits = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
 logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
tmp_eval_accuracy = flat_accuracy(logits, label_ids)
 eval_accuracy += tmp_eval_accuracy
nb_eval_steps += 1
print("Validation Accuracy: {}".format(eval_accuracy/nb_eval_steps))<load_from_csv>
def rmse_cv(model):
rmse = np.sqrt(-cross_val_score(model, X_train, y_train, scoring="neg_mean_squared_error", cv = 10))
return(rmse )
test = pd.read_csv(".. /input/test.tsv", sep='\t')
 test_id = test['PhraseId']<string_transform>
model_ridge = Ridge(alpha = 10 )
sentences = test.Phrase.values
sentences = ["[CLS] " + sentence + " [SEP]" for sentence in sentences]
tokenized_texts = [tokenizer.tokenize(sent)for sent in sentences]<categorify>
model_lasso = make_pipeline(RobustScaler() , Lasso(alpha = 0.0004))
MAX_LEN = 32
input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt)for txt in tokenized_texts],
maxlen=MAX_LEN, dtype="long", truncating="post", padding="post")
 input_ids = [tokenizer.convert_tokens_to_ids(x)for x in tokenized_texts]
input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype="long", truncating="post", padding="post")
 attention_masks = []
for seq in input_ids:
seq_mask = [float(i>0)for i in seq]
attention_masks.append(seq_mask )<define_variables>
cv_ridge = rmse_cv(model_ridge ).mean()
cv_lasso = rmse_cv(model_lasso ).mean()
test['Sentiment'] = 0
labels = test.Sentiment.values<define_variables>
results = pd.DataFrame({
'Model': ['Ridge','Lasso'],
'Score': [cv_ridge,cv_lasso]})
 result_df = results.sort_values(by='Score', ascending=True ).reset_index(drop=True)
 result_df.head()
prediction_inputs = torch.tensor(input_ids)
 prediction_masks = torch.tensor(attention_masks)
 prediction_labels = torch.tensor(labels)
 batch_size = 32<create_dataframe>
model_lasso.fit(X_train, y_train)
 lasso_pred = np.expm1(model_lasso.predict(X_test))
model_ridge.fit(X_train, y_train)
 ridge_pred = np.expm1(model_ridge.predict(X_test))
prediction_data = TensorDataset(prediction_inputs, prediction_masks, prediction_labels)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size )<predict_on_test>
Stack =(lasso_pred + ridge_pred)/ 2
model.eval()
predictions , true_labels = [], []
for batch in prediction_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
logits = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
 logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
predictions.append(logits)
<concatenate>
submission = pd.DataFrame({'Id':test_ID, 'SalePrice':Stack})
 submission.to_csv('submission.csv', index=False )