Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_from_csv>
path = '.. /input/house-prices-advanced-regression-techniques/'
X_full = pd.read_csv(path + 'train.csv', index_col=0)
 test = pd.read_csv(path + 'test.csv', index_col=0)
 target = 'SalePrice'
y = X_full[target]
X = X_full.drop(target, axis=1)
 dataset = [X, test]
X.head()
SEED = 2007
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )<prepare_x_and_y>
def apply(dfs, func, *largs, **kwargs):
for df in dfs:
df = func(df, *largs, **kwargs)
 return dfs
x_train = preprocess(train['comment_text'])
 y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = preprocess(test['comment_text'])
 identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()<data_type_conversions>
def norm(df):
return np.log1p(df)
 def un_norm(df):
return np.expm1(df)
 y = norm(y)
 plot_dist_prob(y, 'Normalized')
 X[target] = y
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T<define_variables>
def drop(df, remove, axis=0):
df.drop(remove, axis=axis, inplace=True)
 return df
mean = y.mean()
std = y.std()
outliers =(y > mean + 3 * std)|(y < mean - 3 * std)
 outliers = outliers[outliers].index
un_norm(y[outliers])
 X = drop(X, outliers, axis=0)
 y = drop(y, outliers, axis=0)
max_features = None<string_transform>
def make_categorical(dfs):
col_transform = ['MSSubClass', 'MoSold']
for col in col_transform:
for df in dfs:
df[col] = df[col].apply(str)
 return dfs
dataset = make_categorical(dataset )
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<define_variables>
def make_drop(df, cols):
return df.drop(cols, axis=1, inplace=True)
 def drop_problematic(dfs):
problem_cols = ['MasVnrType',
'MasVnrArea',
'MiscFeature']
for df in dfs:
df = make_drop(df, problem_cols)
 return dfs
dataset = drop_problematic(dataset )
max_features = max_features or len(tokenizer.word_index)+ 1
max_features<choose_model_class>
def make_impute(df):
df['GarageYrBlt'].fillna(df['YearBuilt'], inplace=True)
 df['Exterior1st'].fillna('Other', inplace=True)
 df['Exterior2nd'].fillna('Other', inplace=True)
 mode_cols = ['MSZoning', 'Electrical', 'SaleType']
for col in mode_cols:
mode = X[col].mode() [0]
df[col].fillna(mode, inplace=True)
 zero_cols = ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF',
'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath',
'GarageCars', 'LotFrontage']
for col in zero_cols:
df[col].fillna(0, inplace=True)
 return df
def impute(dfs):
for df in dfs:
df = make_impute(df)
 return dfs
dataset = impute(dataset )
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))<choose_model_class>
def categorical_to_meaning(dfs):
col_order = [ 
('Alley', ['', 'Grvl', 'Pave']), 
('Utilities', ['', 'ELO', 'NoSeWa', 'NoSewr', 'AllPub']), 
('LotShape', ['Reg', 'IR1', 'IR2', 'IR3']), 
('LandContour', ['Lvl', 'Bnk', 'HLS', 'Low']), 
('LandSlope', ['Gtl', 'Mod', 'Sev']), 
('Fence', ['NA', 'MnWw', 'GdWo', 'MnPrv', 'GdPrv']), 
('PoolQC', ['NA', 'Fa', 'Gd', 'Ex']), 
('PavedDrive', ['N', 'P', 'Y']), 
('GarageCond', ['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('GarageQual', ['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('GarageFinish', ['NA', 'Unf', 'RFn', 'Fin']), 
('GarageType', ['NA', 'Detchd', 'CarPort', 'BuiltIn',
'Basment', 'Attchd', '2Types']), 
('FireplaceQu', ['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('Functional', ['Sal', 'Sev', 'Maj2', 'Maj1', 'Mod',
'Min2', 'Min1', 'Typ']), 
('KitchenQual', ['', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('HeatingQC', ['', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('BsmtFinType1', ['', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ']), 
('BsmtFinType2', ['', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ']), 
('BsmtExposure', ['', 'No', 'Mn', 'Av', 'Gd']), 
('BsmtCond', ['', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('BsmtQual', ['', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('Foundation', ['Wood', 'Stone', 'Slab', 'PConc', 'CBlock', 'BrkTil']), 
('ExterCond', ['', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('ExterQual', ['', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('CentralAir', ['N', 'Y']), 
('Street', ['Grvl', 'Pave']),
]
for co, order in col_order:
imp = order[0]
d = {v: i for i, v in enumerate(order)}
for df in dfs:
df[co] = df[co].fillna(imp ).apply(lambda x: d.get(x, x))
return dfs
dataset = categorical_to_meaning(dataset)
glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))<concatenate>
nans = test['MSZoning'].isna()
test[nans]
X['Electrical'].mode() [0]
X['Electrical'].value_counts()
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()<prepare_x_and_y>
def has_features(dfs):
have = ['GarageArea', 'PoolArea',
'TotalBsmtSF', '2ndFlrSF',
]
for df in dfs:
for col in have:
df['Has'+col] = df[col] > 0
df['has_remod'] = df['YearRemodAdd'] != df['YearBuilt']
df['is_new'] = df['YrSold'] == df['YearBuilt']
return dfs
dataset = has_features(dataset )
x_train_torch = torch.tensor(x_train, dtype=torch.long)
 y_train_torch = torch.tensor(np.hstack([y_train, y_aux_train]), dtype=torch.float32 )<data_type_conversions>
twins_col = ['Fence', 'Fireplaces', 'FireplaceQu',
'PoolArea', 'PoolQC', 'GarageQual',
'GarageArea','GarageCond', '2ndFlrSF',
'BsmtCond', 'TotRmsAbvGrd','GarageFinish',
]
X.drop(twins_col, axis=1, inplace=True)
 test.drop(twins_col, axis=1, inplace=True )
x_test_torch = torch.tensor(x_test, dtype=torch.long )<create_dataframe>
C1, C2 = cols = ['Condition1', 'Condition2']
def condition_weight(val, weights):
return weights.get(val, 0)
 def extract_condition(df, new_features):
for col, wei in new_features.items() :
df[col] = df[C1].apply(
 lambda x: condition_weight(x, wei)
)+ df[C2].apply(
 lambda x: condition_weight(x, wei)
)
 df.drop(cols, axis=1, inplace=True)
 return df
def conditions(dfs):
artery = {'Feedr': 1, 'Artery': 2}
rail = {'RRNe': -2, 'RRNe': -2, 'RRAe': -1, 'RRAn': -1}
posi = {'PosN': 1, 'PosA': 2}
new_features = {
'Artery': artery,
'RailRoad': rail,
'Positive': posi,
}
for df in dfs:
df = extract_condition(df, new_features)
 return dfs
dataset = conditions(dataset )
batch_size = 512
train_dataset = data.TensorDataset(x_train_torch, y_train_torch)
 valid_dataset = data.TensorDataset(x_train_torch[:batch_size], y_train_torch[:batch_size])
 test_dataset = data.TensorDataset(x_test_torch)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)
 databunch = DataBunch(train_dl=train_loader,valid_dl=valid_loader )<compute_test_metric>
def age(df):
df['Age'] = df['YrSold'] - df['YearBuilt']
df.drop(['YearBuilt', 'YearRemodAdd', 'GarageYrBlt'],
axis=1, inplace=True)
 df['YrSold'] = df['YrSold'].apply(str)
 return df
def calc_age(dfs):
for df in dfs:
df = age(df)
 return dfs
dataset = calc_age(dataset )
def custom_loss(data, targets):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ bce_loss_2<choose_model_class>
def summarize(df):
df['total_baths'] = df['BsmtFullBath'] + df['FullBath'] \
+(( df['BsmtHalfBath'] + df['HalfBath'])/ 2)
 df['total_habit'] = df[['TotalBsmtSF', 'GrLivArea']].sum(axis=1)
 porch_cols = ['WoodDeckSF', 'OpenPorchSF',
'EnclosedPorch', '3SsnPorch', 'ScreenPorch']
df['total_porch'] = df[porch_cols].sum(axis=1)
 df.drop(porch_cols, axis=1, inplace=True)
 return df
def sum_rooms(dfs):
for df in dfs:
df = summarize(df)
 return dfs
dataset = sum_rooms(dataset )
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1234 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch,model,loss_func=custom_loss)
 test_preds = train_model(learn,test_dataset,output_dim=7)
 all_test_preds.append(test_preds )<save_to_csv>
def binning(sort):
r = dict()
for i, vals in enumerate(sort):
p = {v: i for v in vals}
r = {**r, **p}
return r
poor = [
'MeadowV', 'IDOTRR', 'BrDale',
]
middle_low = [
'OldTown', 'Edwards','BrkSide',
'Sawyer', 'Blueste', 'SWISU',
'NAmes',
]
middle_med = [
'NPkVill', 'Mitchel',
]
middle_hig = [
'SawyerW', 'Gilbert', 'NWAmes',
'Blmngtn', 'CollgCr', 'ClearCr',
'Crawfor',
'Veenker','Somerst','Timber',
]
high = [
'StoneBr', 'NoRidge', 'NridgHt',
]
sort = [poor, middle_low, middle_med, middle_hig, high]
neig_bin = binning(sort)
 rank_dict = {
'Neighborhood': neig_bin,
}
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )<set_options>
sort = [
['180', '30', '45'],
['190', '50', '90', '85', '40'],
['160', '70', '20', '75', '80', '150'],
['120', '60'],
]
rank_dict['MSSubClass'] = binning(sort )
pd.set_option('display.max_colwidth', 300 )<define_variables>
sort = [
['2fmCon'],
['Duplex', 'Twnhs' ],
['1Fam','TwnhsE'],
]
rank_dict['BldgType'] = binning(sort )
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt']<define_variables>
sort = [
['AdjLand'],
['Abnorml', 'Family'],
['Alloca','Normal'],
[],
['Partial']
]
rank_dict['SaleCondition'] = binning(sort)
 rank_dict['SaleCondition'] = {'AdjLand': 0,
'Abnorml': 1, 'Family': 2,
'Alloca': 3, 'Normal': 4,
'Partial': 5}
NUM_MODELS = 2
BATCH_SIZE = 1024
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4*LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220<define_variables>
def make_rank(df, ranks):
for col, d in ranks.items() :
df[col] = df[col].apply(lambda x: d.get(x, x))
return df
dataset = apply(dataset, make_rank, rank_dict )
IDENTITY_COLUMNS = [
'transgender', 'female', 'homosexual_gay_or_lesbian', 'muslim', 'hindu',
'white', 'black', 'psychiatric_or_mental_illness', 'jewish']<define_variables>
mistakes = {
'Brk Cmn': 'BrkComm',
'CmentBd': 'CemntBd',
}
def correct_mistakes(df, cols, mistake_dict):
for col in cols:
for mistake, correct in mistake_dict.items() :
df[col] = df[col].replace(mistake, correct)
 return df
dataset = apply(dataset, correct_mistakes, ecols, mistakes )
AUX_COLUMNS = ['target', 'severe_toxicity','obscene','identity_attack','insult','threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'<load_pretrained>
def make_exterior(df, values):
E1, E2 = ecols = ['Exterior1st', 'Exterior2nd']
for v in values:
df['Exterior_' + v] =(df[E1] == v)|(df[E2] == v)
 df = df.drop(ecols, axis=1, inplace=True)
 return df
values = set(X[E1].unique()
).union(set(X[E2].unique()))
dataset = apply(dataset, make_exterior, values )
contractions = joblib.load('.. /input/contractions/contraction_mapping' )<load_from_csv>
noise_cols = ['LotConfig', 'RoofStyle', 'RoofMatl',
'Heating', 'Electrical', 'SaleType',
]
dataset = apply(dataset, make_drop, noise_cols)
swear_words = pd.read_table('.. /input/swear-list/swear_wordlist.txt')
 swear_words.columns = ['words']
swear_words['words'] = swear_words['words'].map(lambda x: x.strip())
 swear_words = swear_words['words'].tolist()<string_transform>
X = pd.get_dummies(X)
 test = pd.get_dummies(test)
 print(list(X.columns))
replace_with_fuck = []
for swear in swear_words:
replace_with_fuck.append(swear)
 replace_with_fuck = '|'.join(replace_with_fuck )<string_transform>
row = X.shape[0]
overfit_min = row - math.ceil(row * 5 / 10000)
 c =(X == 0 ).sum() >= overfit_min
overfit_cols = set(c[c].index)
 X.drop(overfit_cols, axis=1, inplace=True)
 both = set(test.columns ).intersection(overfit_cols)
 test.drop(both, axis=1, inplace=True)
 test_overfit = list(set(X.columns)- set(test.columns))
if test_overfit:
X.drop(test_overfit, axis=1, inplace=True)
 if target in X.columns:
X.drop([target], axis=1, inplace=True )
def handle_swears(text):
text = re.sub(replace_with_fuck, ' fuck ', text)
 return text
def clean_sentence(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 text = handle_swears(text)
 wordnet_lemmatizer = WordNetLemmatizer()
stop = set(stopwords.words('english'))
VERB_CODES = {'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ', }
ADJ_CODES = {'JJ','JJR','JJS'}
ADV_CODES = {'RB','RBR','RBS'}
NOUN_CODES = {'NN', 'NNP','NNS','NNPS'}
new_sentence = []
for i, word in enumerate(words):
if role[i][1] in VERB_CODES:
new_word = wordnet_lemmatizer.lemmatize(word, 'v')
 else:
if role[i][1] in ADJ_CODES:
new_word = wordnet_lemmatizer.lemmatize(word, 'a')
 else:
if role[i][1] in ADV_CODES:
new_word = wordnet_lemmatizer.lemmatize(word, 'r')
 else:
if role[i][1] in NOUN_CODES:
new_word = wordnet_lemmatizer.lemmatize(word, 'n')
 else:
new_word = wordnet_lemmatizer.lemmatize(word)
 if new_word not in stop and new_word.isalpha() and len(new_word)>1:
new_sentence.append(new_word)
 s = ' '.join(new_sentence)
 return s
def get_coefs(word, *arr):

 return word, np.asarray(arr,dtype='float32')
<load_pretrained>
scaler = RobustScaler(with_centering=True)
 X_scale = pd.DataFrame(scaler.fit_transform(X),
columns=X.columns)
 test_scale = pd.DataFrame(scaler.transform(test),
columns=test.columns )
def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f )<categorify>
def RMSE(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
SEED = 2007
common_args = {
'random_state': SEED,
}
def build_matrix(word_index, path):

 embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix<choose_model_class>
cv = KFold(n_splits=10, shuffle=True, random_state=SEED)
 def cross_val(models, X=X, y=y):
r = dict()
for name, model in models.items() :
cv_results = np.sqrt(
 -cross_val_score(model, X, y,
cv=cv,
scoring='neg_mean_squared_error')
)
 r[name] = cv_results
print('Cross validation ready!')
 return r
def choose_best(results):
errors = dict()
for name, arr in results.items() :
errors[name] = arr.mean()
print(name, 'RMSE {0:.4f}, Std {1:.4f}'.format(
 errors[name], arr.std()))
best_model = [m for m, e in errors.items()
if e == min(errors.values())][0]
return best_model
def build_model(embedding_matrix, num_aux_targets):

 words = Input(shape=(MAX_LEN,))
x = Embedding(*embedding_matrix.shape, weights = [embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x)
 ])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result =Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs =words, outputs =[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<load_from_csv>
def output(predicts, filename):
output = pd.DataFrame({'Id': test.index,
'SalePrice': predicts})
 output.to_csv(filename + '.csv', index=False)
 def train_predict_out(model, filename, X=X, y=y, test=test):
model.fit(X, y)
 predicts = un_norm(model.predict(test))
output(predicts, filename)
 return predicts
train_df = pd.read_csv('.. /input/cleaned/cleaned_train_df.csv')
 test_df = pd.read_csv('.. /input/cleaned/cleaned_test_df.csv' )<feature_engineering>
lin_models = {
'Lasso': LassoCV(cv=cv,
**common_args 
),
'LassoLars': LassoLarsCV(cv=cv 
),
}
r = cross_val(lin_models, X_scale)
 best_linear = choose_best(r)
 linear_model = lin_models[best_linear]
<feature_engineering>
tree_models = {
'Random Forest': RandomForestRegressor(n_estimators=100,
max_depth=10,
max_features='sqrt',
warm_start=False,
**common_args),
'XGBoost': XGBRegressor(n_estimators=2000,
max_depth=3,
learning_rate=0.01,
verbosity=0,
objective='reg:squarederror',
**common_args),
}
results = cross_val(tree_models, X)
 best_treemodel = choose_best(results)
 tree_model = tree_models[best_treemodel]
<set_options><EOS> 
lin_predicts = train_predict_out(linear_model, 'linear',
X=X_scale, test=test_scale)
 tre_predicts = train_predict_out(tree_model, 'tree')
 blend_predicts =(lin_predicts + tre_predicts)/ 2
output(blend_predicts, 'blended' )