Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<feature_engineering>
%matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
def dice_overall(preds, targs):
n = preds.shape[0]
preds = preds.view(n, -1)
 targs = targs.view(n, -1)
 intersect =(preds * targs ).sum(-1 ).float()
union =(preds+targs ).sum(-1 ).float()
u0 = union==0
intersect[u0] = 1
union[u0] = 2
return(2.* intersect / union )<predict_on_test>
test = pd.read_csv(".. /input/test.csv", index_col = 0)
 train = pd.read_csv(".. /input/train.csv", index_col = 0)
 train.head()
def pred_with_flip(learn:fastai.basic_train.Learner,
ds_type:fastai.basic_data.DatasetType=DatasetType.Valid):
preds, ys = learn.get_preds(ds_type)
 preds = preds[:,1,...]
learn.data.dl(ds_type ).dl.dataset.tfms.append(flip_lr())
 preds_lr, ys = learn.get_preds(ds_type)
 del learn.data.dl(ds_type ).dl.dataset.tfms[-1]
preds_lr = preds_lr[:,1,...]
ys = ys.squeeze()
preds = 0.5*(preds + torch.flip(preds_lr,[-1]))
del preds_lr
gc.collect()
torch.cuda.empty_cache()
return preds, ys<categorify>
na_ratios = all_data.isnull().sum() / len(all_data)
 na_ratios = na_ratios.drop(na_ratios[na_ratios == 0].index ).sort_values(ascending = False)
 na_ratios = pd.DataFrame({"Missing Proportion": na_ratios})
 na_ratios = na_ratios.drop(na_ratios[na_ratios["Missing Proportion"] <0.005].index)
 na_ratios
class SegmentationLabelList(SegmentationLabelList):
def open(self, fn): return open_mask(fn, div=True)
 class SegmentationItemList(SegmentationItemList):
_label_cls = SegmentationLabelList
def transform(self, tfms:Optional[Tuple[TfmList,TfmList]]=(None,None), **kwargs):
if not tfms: tfms=(None,None)
 assert is_listy(tfms)and len(tfms)== 2
self.train.transform(tfms[0], **kwargs)
 self.valid.transform(tfms[1], **kwargs)
 kwargs['tfm_y'] = False
if self.test: self.test.transform(tfms[1], **kwargs)
 return self
fastai.data_block.ItemLists.transform = transform<categorify>
na_ratios = all_data.isnull().sum()
na_ratios = na_ratios.drop(na_ratios[na_ratios == 0].index)
 na_ratios = pd.DataFrame({"Missing Count": na_ratios})
 print(na_ratios)
 na_ratios.to_csv("NA_Ticklist.csv" )
def get_data(fold):
kf = KFold(n_splits=nfolds, shuffle=True, random_state=SEED)
 valid_idx = list(kf.split(list(range(len(Path(TRAIN ).ls())))))[fold][1]
data =(SegmentationItemList.from_folder(TRAIN)
.split_by_idx(valid_idx)
.label_from_func(lambda x : str(x ).replace('train', 'masks'), classes=[0,1])
.add_test(Path(TEST ).ls() , label=None)
.transform(get_transforms() , size=sz, tfm_y=True)
.databunch(path=Path('.'), bs=bs)
.normalize(stats))
return data
get_data(0 ).show_batch()<load_pretrained>
all_data["MSZoning"] = all_data["MSZoning"].fillna("RL")
 all_data["MSZoning"].value_counts()
unet_learner(get_data(0), models.resnet34, metrics=[dice] ).model<train_model>
lf_na_index = all_data["LotFrontage"][all_data["LotFrontage"].isnull() ].index
lf_na_ests = all_data["MSSubClass"][lf_na_index].map(lf_est)
 all_data["LotFrontage"][lf_na_index] = lf_na_ests
all_data[["LotFrontage", "MSSubClass"]].ix[lf_na_index].head()
all_data["LotFrontage"] = all_data["LotFrontage"].fillna(np.nanmedian(all_data["LotFrontage"]))
scores, best_thrs = [],[]
for fold in range(nfolds):
print('fold: ', fold)
 data = get_data(fold)
 learn = unet_learner(data, models.resnet34, metrics=[dice])
 learn.clip_grad(1.0);
set_BN_momentum(learn.model)
 lr = 1e-3
learn.fit_one_cycle(6, lr, callbacks = [AccumulateStep(learn,n_acc)])
 learn.unfreeze()
learn.fit_one_cycle(12, slice(lr/80, lr/2), callbacks=[AccumulateStep(learn,n_acc)])
 learn.save('fold'+str(fold)) ;
preds, ys = pred_with_flip(learn)
 pt, _ = pred_with_flip(learn,DatasetType.Test)
 if fold == 0: preds_test = pt
else: preds_test += pt
preds_save =(preds*255.0 ).byte()
torch.save(preds_save, 'preds_fold'+str(fold)+'.pt')
 np.save('items_fold'+str(fold), data.valid_ds.items)
 preds[preds.view(preds.shape[0],-1 ).sum(-1)< noise_th,...] = 0.0
dices = []
thrs = np.arange(0.01, 1, 0.01)
 for th in progress_bar(thrs):
preds_m =(preds>th ).long()
dices.append(dice_overall(preds_m, ys ).mean())
 dices = np.array(dices)
 scores.append(dices.max())
 best_thrs.append(thrs[dices.argmax() ])
 if fold != nfolds-1: del preds, ys, preds_save
gc.collect()
torch.cuda.empty_cache()
preds_test /= nfolds<compute_test_metric>
na_means_none_features = ["Alley", "BsmtQual", "BsmtCond", "BsmtExposure", "BsmtFinType1",
"BsmtFinType2", "FireplaceQu", "GarageType", "GarageFinish", "GarageQual",
"GarageCond", "PoolQC", "Fence", "MiscFeature"]
all_data[na_means_none_features] = all_data[na_means_none_features].fillna("None" )
print('scores: ', scores)
 print('mean score: ', np.array(scores ).mean())
 print('thresholds: ', best_thrs)
 best_thr = np.array(best_thrs ).mean()
print('best threshold: ', best_thr )<feature_engineering>
all_data["Utilities"] = all_data["Utilities"].fillna("AllPub")
 all_data["Utilities"].value_counts()
preds_save =(preds_test*255.0 ).byte()
torch.save(preds_save, 'preds_test.pt')
 preds_test[preds_test.view(preds_test.shape[0],-1 ).sum(-1)< noise_th,...] = 0.0<categorify>
all_data["Exterior1st"] = all_data["Exterior1st"].fillna("VinylSd")
 all_data["Exterior1st"].value_counts()
preds_test =(preds_test>best_thr ).long().numpy()
rles = []
for p in progress_bar(preds_test):
im = PIL.Image.fromarray(( p.T*255 ).astype(np.uint8)).resize(( 1024,1024))
im = np.asarray(im)
 rles.append(mask2rle(im, 1024, 1024))<save_to_csv>
all_data["Exterior2nd"] = all_data["Exterior2nd"].fillna("VinylSd")
 all_data["Exterior2nd"].value_counts()
ids = [o.stem for o in data.test_ds.items]
sub_df = pd.DataFrame({'ImageId': ids, 'EncodedPixels': rles})
 sub_df.loc[sub_df.EncodedPixels=='', 'EncodedPixels'] = '-1'
sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<load_from_csv>
fill_feature = "MasVnrType"
all_data[fill_feature] = all_data[fill_feature].fillna("None")
 all_data[fill_feature].value_counts()
fill_feature = "MasVnrArea"
all_data[fill_feature] = all_data[fill_feature].fillna(0 )
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 display(train)
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 display(test )<drop_column>
fill_feature = ["BsmtFinSF1", "BsmtFinSF2", "BsmtUnfSF", "BsmtFullBath", "BsmtHalfBath", "TotalBsmtSF"]
all_data[fill_feature] = all_data[fill_feature].fillna(0)
 all_data[fill_feature].head()
train.drop(
 [
6449, 7034, 3589, 3591, 3597, 3600, 3603,
3604, 3610, 3613, 3614, 119, 106, 115,
2666, 2679, 1356, 7609, 3382, 1335, 2655,
2674, 1343, 4291, 4303, 1345, 48, 3374,
7600, 164, 5292, 2352, 4308, 4306, 4310,
1332, 1156, 7610, 2441, 2449, 2454, 2477,
2452, 2456, 3390, 7611, 6656, 1360, 5771,
4351, 5073, 4601, 5665, 7135, 5720, 5723,
5734, 1623, 7533, 7537, 7026, 4834, 4631,
3461, 6366, 6373, 6377, 6378, 6392, 2828,
2841, 1725, 3795, 1251, 7607
], inplace=True 
)
 duplicates = pd.concat(x for _, x in train.groupby(["text"])if len(x)> 1)
 with pd.option_context("display.max_rows", None, "max_colwidth", 240):
display(duplicates[["id", "target", "text"]] )<drop_column>
fill_feature = "SaleType"
all_data[fill_feature] = all_data[fill_feature].fillna("WD")
 all_data[fill_feature].value_counts()
train.drop(
 [
4290, 4299, 4312, 4221, 4239, 4244, 2830,
2831, 2832, 2833, 4597, 4605, 4618, 4232,
4235, 3240, 3243, 3248, 3251, 3261, 3266,
4285, 4305, 4313, 1214, 1365, 6614, 6616,
1197, 1331, 4379, 4381, 4284, 4286, 4292,
4304, 4309, 4318, 610, 624, 630, 634, 3985,
4013, 4019, 1221, 1349, 6091, 6094,
6103, 6123, 5620, 5641
], inplace=True
)<categorify>
fill_feature = "Electrical"
all_data[fill_feature] = all_data[fill_feature].fillna("SBrkr")
 all_data[fill_feature].value_counts()
def fix_text_issues(x):
x = x.lower()
x = x.replace("&amp;", "and")
 x = x.replace("&lt;", "<")
 x = x.replace("&gt;", ">")
 x = re.sub("(\W|^)hwy\.( \W)", "\\1highway\\2", x)
 x = re.sub("(\W|^)ave.( \W)", "\\1avenue\\2", x)
 x = re.sub("(\W|^)fyi(\W)", "\\1for your information\\2", x)
 x = re.sub("(\W|^)ain't(\W)", "\\1am not\\2", x)
 x = re.sub("(\W|^)can't(\W)", "\\1cannot\\2", x)
 x = re.sub("(\W|^)cant(\W)", "\\1cannot\\2", x)
 x = re.sub("(\W|^)rt(\W)", "\\1retweet\\2", x)
 x = x.replace("g'day", "good day")
 x = x.replace("giv'n", "given")
 x = x.replace("let's", "let us")
 x = x.replace("ma'am", "madam")
 x = x.replace("ne'er", "never")
 x = x.replace("o'clock", "of the clock")
 x = x.replace("o'er", "over")
 x = x.replace("ol'", "old")
 x = x.replace("shan't", "shall not")
 x = x.replace("y'all", "you all")
 x = x.replace("'tis", "it is")
 x = x.replace("'hood", "neighborhood")
 x = x.replace("can ªt", "cannot")
 x = x.replace("åÊ", " ")
 x = x.replace("ÛÏ", " ")
 x = x.replace("?Û", " ")
 x = x.replace("Û_", " ")
 x = x.replace("Û÷", " ")
 x = x.replace("ûò", " ")
 x = x.replace("û", " ")
 x = re.sub("\W'twas", " it was", x)
 x = re.sub("\W'cause", " because", x)
 x = re.sub("(\w)'ve", "\\1 have", x)
 x = re.sub("(\w)n't", "\\1 not", x)
 x = re.sub("(\w)'s", "\\1 is", x)
 x = re.sub("(\w)'d", "\\1 had", x)
 x = re.sub("(\w)'ll", "\\1 will", x)
 x = re.sub("(\w)'re", "\\1 are", x)
 x = re.sub("(\w)'m", "\\1 am", x)
 x = re.sub("http[s]*://t.co/\S+", "", x)
 x = x.replace("...", " ")
 x = strip_multiple_whitespaces(x)
 x = strip_punctuation(x)
 x = stem_text(x)
 return x.strip()
def clean_text(df):
df["text"] = df["text"].apply(fix_text_issues)
 clean_text(train)
 clean_text(test )<set_options>
fill_feature = "KitchenQual"
all_data[fill_feature] = all_data[fill_feature].fillna("TA")
 all_data[fill_feature].value_counts()
pipeline = Pipeline([('tfidf', TfidfVectorizer(decode_error="ignore")) ,('clf', SVC(random_state=2020)) ])
 parameters = {
'tfidf__ngram_range':(( 1,1),(1,2),(2,2)) ,
'tfidf__use_idf':(True, False),
'tfidf__smooth_idf':(True, False),
'tfidf__sublinear_tf':(True, False),
'clf__C':(1.5, 1.7, 1.9),
}
grid = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=3, cv=3)
 grid_result = grid.fit(train["text"], train["target"])
 print("Best score: {:0.5f}".format(grid_result.best_score_))
print("Best parameters: {}".format(grid_result.best_params_))<find_best_model_class>
fill_feature = "Functional"
all_data[fill_feature] = all_data[fill_feature].fillna("Typ")
 all_data[fill_feature].value_counts()
x_train, x_valid, y_train, y_valid = train_test_split(
 train["text"], train["target"], test_size=0.2, random_state=2020 
)
 vectorizer = TfidfVectorizer(
 decode_error="ignore",
ngram_range=(1,2),
smooth_idf=False,
sublinear_tf=True,
use_idf=True 
)
 x_train_tfidf = vectorizer.fit_transform(x_train)
 x_valid_tfidf = vectorizer.transform(x_valid)
 model = SVC(random_state=2020, C=1.7)
 model.fit(x_train_tfidf, y_train)
 train_predictions = model.predict(x_valid_tfidf)
 print(classification_report(y_valid, train_predictions, target_names=["Not Real", "Real"]))
score = model.score(x_valid_tfidf, y_valid)
 print("--> Mean accuracy {:0.5}".format(score))
print("" )<feature_engineering>
fill_feature = ["GarageCars", "GarageArea"]
all_data[fill_feature] = all_data[fill_feature].fillna(0)
 all_data[fill_feature].head()
vectorizer = TfidfVectorizer(
 decode_error="ignore",
ngram_range=(1,2),
smooth_idf=False,
sublinear_tf=True,
use_idf=True 
)
 train_tfidf = vectorizer.fit_transform(train["text"])
 model = SVC(random_state=2020, C=1.7)
 model.fit(train_tfidf, train["target"] )<save_to_csv>
garageYrMap = all_data.pivot_table("GarageYrBlt", "OverallQual")
 garageYrMap = pd.Series(garageYrMap["GarageYrBlt"],
index = garageYrMap.index)
 qual_to_garage_map = all_data["OverallQual"].map(garageYrMap)
 garageYrBlt_nas = all_data["GarageYrBlt"].isnull().index
all_data["GarageYrBlt"][all_data["GarageYrBlt"].isnull() ] = qual_to_garage_map[all_data["GarageYrBlt"].isnull() ]
garageYrMap
clean_text(test)
 test_tfidf = vectorizer.transform(test["text"])
 predictions = model.predict(test_tfidf)
 submission = pd.DataFrame({"id": test["id"], "target": predictions})
 submission.to_csv("submission.csv", index=False )<load_from_url>
na_ratios = all_data.isnull().sum()
na_ratios = na_ratios.drop(na_ratios[na_ratios == 0].index)
 na_ratios = pd.DataFrame({"Missing Count": na_ratios})
 print(na_ratios )
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<import_modules>
numerical_features = ['LotFrontage',
'LotArea',
'OverallQual',
'YearBuilt',
'MasVnrArea',
'BsmtFinSF1',
'BsmtFinSF2',
'BsmtUnfSF',
'TotalBsmtSF',
'1stFlrSF',
'2ndFlrSF',
'GrLivArea',
'FullBath',
'TotRmsAbvGrd',
'Fireplaces',
'GarageYrBlt',
'GarageCars',
'GarageArea',
'WoodDeckSF',
'OpenPorchSF',
'EnclosedPorch',
'ScreenPorch',
'TotalSF']
unordered_cat_features = ['MSSubClass',
'MSZoning',
'Street',
'Alley',
'LotShape',
'LandContour',
'LotConfig',
'LandSlope',
'Neighborhood',
'Condition1',
'Condition2',
'BldgType',
'HouseStyle',
'OverallCond',
'RoofStyle',
'RoofMatl',
'Exterior1st',
'Exterior2nd',
'MasVnrType',
'ExterQual',
'ExterCond',
'Foundation',
'BsmtQual',
'BsmtCond',
'BsmtExposure',
'BsmtFinType1',
'BsmtFinType2',
'Heating',
'HeatingQC',
'CentralAir',
'Electrical',
'LowQualFinSF',
'BsmtFullBath',
'BsmtHalfBath',
'HalfBath',
'BedroomAbvGr',
'KitchenAbvGr',
'KitchenQual',
'Functional',
'FireplaceQu',
'GarageType',
'GarageFinish',
'GarageQual',
'GarageCond',
'PavedDrive',
'3SsnPorch',
'PoolArea',
'PoolQC',
'Fence',
'MiscFeature',
'MiscVal',
'MoSold',
'YrSold',
'SaleType',
'SaleCondition']
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization<categorify>
all_data['MSZoning'] = all_data['MSZoning'].apply(str )
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<choose_model_class>
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 cat_pivot("MSSubClass" )
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<choose_model_class>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam)
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<load_from_csv>
all_numerical_features = all_data[numerical_features]
unordCatOHE = OneHotEncoder(sparse = False)
 all_categorical_features = pd.DataFrame(unordCatOHE.fit_transform(all_data[unordered_cat_features]), index = all_data.index)
 all_data_features = pd.concat([all_numerical_features,
all_categorical_features],
axis = 1)
 X_train = all_data_features.ix[train.index]
X_test = all_data_features.ix[test.index]
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )<feature_engineering>
def preds_to_output(preds):

 output = pd.DataFrame({
"Id": X_test.index,
"SalePrice": preds
})
 output.to_csv("submission.csv", index = False)
 print("CSV written successfully" )
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
rtree = RandomForestRegressor()
fitted_model = rtree.fit(X_train, y_train)
 preds = np.expm1(rtree.predict(X_test))
def cross_val(model):
return -np.mean(cross_val_score(model, X_train, y_train, cv = 5, scoring = "neg_mean_absolute_error"))
cross_val(rtree )
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values<train_model>
model = XGBRegressor(n_estimators = 3000,
learning_rate = 0.005,
max_depth = 5,
colsample_bytree = 0.3,
gamma = 0)
 model.fit(X_train, y_train)
 cross_val(model)
 xgb_preds = np.expm1(model.predict(X_test))
preds_to_output(xgb_preds )
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
callbacks=[checkpoint],
batch_size=16
)<predict_on_test>
robScale = RobustScaler()
robScaleXtrain = robScale.fit_transform(X_train)
 robScaleXtest = robScale.transform(X_test)
 model=Lasso(alpha = 0.00001, random_state=1)
 model.fit(robScaleXtrain, y_train)
 lasso_preds = np.expm1(model.predict(robScaleXtest))
preds_to_output(lasso_preds )
model.load_weights('model.h5')
 test_pred = model.predict(test_input )<save_to_csv>
preds_to_output(((lasso_preds + xgb_preds)/ 2))