Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<drop_column>
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
df['comment_text'] = df['comment_text'].apply(lambda x: clean_contractions(x, contraction_mapping))<compute_train_metric>
train=pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test=pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv" )
vocab = build_vocab(df['comment_text'])
 oov = check_coverage(vocab, embeddings_index)
 oov[:10]<set_options>
train_ID=train['Id']
test_ID=test['Id']
train.drop("Id",axis=1,inplace=True)
 test.drop("Id",axis=1,inplace=True )
del(vocab,oov)
 gc.collect()<drop_column>
train=train[train["GrLivArea"]<4500]
train.reset_index(drop=True,inplace=True )
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown<define_variables>
train['SalePrice']=np.log1p(train['SalePrice'])
 y=train['SalePrice']
train_features=train.drop('SalePrice',axis=1)
 test_features=test
punct_mapping = {"_":" ", "`":" "}<categorify>
features=pd.concat([train_features,test_features],axis=0)
 numeric_t = [f for f in features.columns if features.dtypes[f] != 'object']
char_t = [f for f in features.columns if features.dtypes[f] == 'object']
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 return text<feature_engineering>
for col in numeric_t:
if features[col].isnull().sum() >0:
print("{} is lack of {}".format(col,features[col].isnull().sum()))
df['comment_text'] = df['comment_text'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))<compute_train_metric>
for col in char_t:
if features[col].isnull().sum() >0:
print("{} is lack of {}".format(col,features[col].isnull().sum()))
vocab = build_vocab(df['comment_text'])
 oov = check_coverage(vocab, embeddings_index)
<set_options>
features['MSSubClass'] = features['MSSubClass'].astype(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str)
 features['Functional']=features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['Exterior1st']=features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd']=features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
 features["PoolQC"] = features["PoolQC"].fillna("None")
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
del(vocab,oov)
 gc.collect()<define_variables>
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics.append(i)
 features.update(features[numerics].fillna(0))
numerics2 = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics2.append(i)
swear_words = [
' 4r5e ',
' 5h1t ',
' 5hit ',
' a55 ',
' anal ',
' anus ',
' ar5e ',
' arrse ',
' arse ',
' ass ',
' ass-fucker ',
' asses ',
' assfucker ',
' assfukka ',
' asshole ',
' assholes ',
' asswhole ',
' a_s_s ',
' b!tch ',
' b00bs ',
' b17ch ',
' b1tch ',
' ballbag ',
' balls ',
' ballsack ',
' bastard ',
' beastial ',
' beastiality ',
' bellend ',
' bestial ',
' bestiality ',
' biatch ',
' bitch ',
' bitcher ',
' bitchers ',
' bitches ',
' bitchin ',
' bitching ',
' bloody ',
' blow job ',
' blowjob ',
' blowjobs ',
' boiolas ',
' bollock ',
' bollok ',
' boner ',
' boob ',
' boobs ',
' booobs ',
' boooobs ',
' booooobs ',
' booooooobs ',
' breasts ',
' buceta ',
' bugger ',
' bum ',
' bunny fucker ',
' butt ',
' butthole ',
' buttmuch ',
' buttplug ',
' c0ck ',
' c0cksucker ',
' carpet muncher ',
' cawk ',
' chink ',
' cipa ',
' cl1t ',
' clit ',
' clitoris ',
' clits ',
' cnut ',
' cock ',
' cock-sucker ',
' cockface ',
' cockhead ',
' cockmunch ',
' cockmuncher ',
' cocks ',
' cocksuck ',
' cocksucked ',
' cocksucker ',
' cocksucking ',
' cocksucks ',
' cocksuka ',
' cocksukka ',
' cok ',
' cokmuncher ',
' coksucka ',
' coon ',
' cox ',
' crap ',
' cum ',
' cummer ',
' cumming ',
' cums ',
' cumshot ',
' cunilingus ',
' cunillingus ',
' cunnilingus ',
' cunt ',
' cuntlick ',
' cuntlicker ',
' cuntlicking ',
' cunts ',
' cyalis ',
' cyberfuc ',
' cyberfuck ',
' cyberfucked ',
' cyberfucker ',
' cyberfuckers ',
' cyberfucking ',
' d1ck ',
' damn ',
' dick ',
' dickhead ',
' dildo ',
' dildos ',
' dink ',
' dinks ',
' dirsa ',
' dlck ',
' dog-fucker ',
' doggin ',
' dogging ',
' donkeyribber ',
' doosh ',
' duche ',
' dyke ',
' ejaculate ',
' ejaculated ',
' ejaculates ',
' ejaculating ',
' ejaculatings ',
' ejaculation ',
' ejakulate ',
' f u c k ',
' f u c k e r ',
' f4nny ',
' fag ',
' fagging ',
' faggitt ',
' faggot ',
' faggs ',
' fagot ',
' fagots ',
' fags ',
' fanny ',
' fannyflaps ',
' fannyfucker ',
' fanyy ',
' fatass ',
' fcuk ',
' fcuker ',
' fcuking ',
' feck ',
' fecker ',
' felching ',
' fellate ',
' fellatio ',
' fingerfuck ',
' fingerfucked ',
' fingerfucker ',
' fingerfuckers ',
' fingerfucking ',
' fingerfucks ',
' fistfuck ',
' fistfucked ',
' fistfucker ',
' fistfuckers ',
' fistfucking ',
' fistfuckings ',
' fistfucks ',
' flange ',
' fook ',
' fooker ',
' fuck ',
' fucka ',
' fucked ',
' fucker ',
' fuckers ',
' fuckhead ',
' fuckheads ',
' fuckin ',
' fucking ',
' fuckings ',
' fuckingshitmotherfucker ',
' fuckme ',
' fucks ',
' fuckwhit ',
' fuckwit ',
' fudge packer ',
' fudgepacker ',
' fuk ',
' fuker ',
' fukker ',
' fukkin ',
' fuks ',
' fukwhit ',
' fukwit ',
' fux ',
' fux0r ',
' f_u_c_k ',
' gangbang ',
' gangbanged ',
' gangbangs ',
' gaylord ',
' gaysex ',
' goatse ',
' God ',
' god-dam ',
' god-damned ',
' goddamn ',
' goddamned ',
' hardcoresex ',
' hell ',
' heshe ',
' hoar ',
' hoare ',
' hoer ',
' homo ',
' hore ',
' horniest ',
' horny ',
' hotsex ',
' jack-off ',
' jackoff ',
' jap ',
' jerk-off ',
' jism ',
' jiz ',
' jizm ',
' jizz ',
' kawk ',
' knob ',
' knobead ',
' knobed ',
' knobend ',
' knobhead ',
' knobjocky ',
' knobjokey ',
' kock ',
' kondum ',
' kondums ',
' kum ',
' kummer ',
' kumming ',
' kums ',
' kunilingus ',
' l3itch ',
' labia ',
' lmfao ',
' lust ',
' lusting ',
' m0f0 ',
' m0fo ',
' m45terbate ',
' ma5terb8 ',
' ma5terbate ',
' masochist ',
' master-bate ',
' masterb8 ',
' masterbat3 ',
' masterbate ',
' masterbation ',
' masterbations ',
' masturbate ',
' mo-fo ',
' mof0 ',
' mofo ',
' mothafuck ',
' mothafucka ',
' mothafuckas ',
' mothafuckaz ',
' mothafucked ',
' mothafucker ',
' mothafuckers ',
' mothafuckin ',
' mothafucking ',
' mothafuckings ',
' mothafucks ',
' mother fucker ',
' motherfuck ',
' motherfucked ',
' motherfucker ',
' motherfuckers ',
' motherfuckin ',
' motherfucking ',
' motherfuckings ',
' motherfuckka ',
' motherfucks ',
' muff ',
' mutha ',
' muthafecker ',
' muthafuckker ',
' muther ',
' mutherfucker ',
' n1gga ',
' n1gger ',
' nazi ',
' nigg3r ',
' nigg4h ',
' nigga ',
' niggah ',
' niggas ',
' niggaz ',
' nigger ',
' niggers ',
' nob ',
' nob jokey ',
' nobhead ',
' nobjocky ',
' nobjokey ',
' numbnuts ',
' nutsack ',
' orgasim ',
' orgasims ',
' orgasm ',
' orgasms ',
' p0rn ',
' pawn ',
' pecker ',
' penis ',
' penisfucker ',
' phonesex ',
' phuck ',
' phuk ',
' phuked ',
' phuking ',
' phukked ',
' phukking ',
' phuks ',
' phuq ',
' pigfucker ',
' pimpis ',
' piss ',
' pissed ',
' pisser ',
' pissers ',
' pisses ',
' pissflaps ',
' pissin ',
' pissing ',
' pissoff ',
' poop ',
' porn ',
' porno ',
' pornography ',
' pornos ',
' prick ',
' pricks ',
' pron ',
' pube ',
' pusse ',
' pussi ',
' pussies ',
' pussy ',
' pussys ',
' rectum ',
' retard ',
' rimjaw ',
' rimming ',
' s hit ',
' s.o.b.',
' sadist ',
' schlong ',
' screwing ',
' scroat ',
' scrote ',
' scrotum ',
' semen ',
' sex ',
' sh!t ',
' sh1t ',
' shag ',
' shagger ',
' shaggin ',
' shagging ',
' shemale ',
' shit ',
' shitdick ',
' shite ',
' shited ',
' shitey ',
' shitfuck ',
' shitfull ',
' shithead ',
' shiting ',
' shitings ',
' shits ',
' shitted ',
' shitter ',
' shitters ',
' shitting ',
' shittings ',
' shitty ',
' skank ',
' slut ',
' sluts ',
' smegma ',
' smut ',
' snatch ',
' son-of-a-bitch ',
' spac ',
' spunk ',
' s_h_i_t ',
' t1tt1e5 ',
' t1tties ',
' teets ',
' teez ',
' testical ',
' testicle ',
' tit ',
' titfuck ',
' tits ',
' titt ',
' tittie5 ',
' tittiefucker ',
' titties ',
' tittyfuck ',
' tittywank ',
' titwank ',
' tosser ',
' turd ',
' tw4t ',
' twat ',
' twathead ',
' twatty ',
' twunt ',
' twunter ',
' v14gra ',
' v1gra ',
' vagina ',
' viagra ',
' vulva ',
' w00se ',
' wang ',
' wank ',
' wanker ',
' wanky ',
' whoar ',
' whore ',
' willies ',
' willy ',
' xrated ',
' xxx '
]<string_transform>
skew_features = features[numerics2].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))
replace_with_fuck = []
for swear in swear_words:
if swear[1:(len(swear)-1)] not in embeddings_index:
replace_with_fuck.append(swear)
 replace_with_fuck = '|'.join(replace_with_fuck)
 replace_with_fuck 
<categorify>
features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['Total_sqr_footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['Total_porch_sf'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
 features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )
def handle_swears(text):
text = re.sub(replace_with_fuck, ' fuck ', text)
 return text<feature_engineering>
final_features = pd.get_dummies(features ).reset_index(drop=True)
 print(final_features.shape)
 X = final_features.iloc[:len(y), :]
X_sub = final_features.iloc[len(X):, :]
print('X', X.shape, 'y', y.shape, 'X_sub', X_sub.shape)
 outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers] )
df['comment_text'] = df['comment_text'].apply(lambda x: handle_swears(x))
gc.collect()<split>
overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 overfit.append('MSZoning_C(all)')
 X = X.drop(overfit, axis=1 ).copy()
X_sub = X_sub.drop(overfit, axis=1 ).copy()
train = df.iloc[:1804874,:]
test = df.iloc[1804874:,:]
train.head()<set_options>
import numpy as np
import pandas as pd
from datetime import datetime
from scipy.stats import skew
from scipy.special import boxcox1p
from scipy.stats import boxcox_normmax
from sklearn.linear_model import ElasticNetCV, LassoCV, RidgeCV
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.svm import SVR
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error
from mlxtend.regressor import StackingCVRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
del(df)
 gc.collect()<load_from_csv>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y,
scoring="neg_mean_squared_error",
cv=kfolds))
return(rmse )
train_orig = pd.read_csv(".. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv")
 train_orig.head()<concatenate>
kfolds=KFold(n_splits=10,shuffle=True,random_state=42)
 scale=RobustScaler().fit(X)
 X1=scale.transform(X)
 model=Ridge()
rid_param_grid = {"alpha":[19.8]}
grid_search= GridSearchCV(model,param_grid=rid_param_grid,cv=kfolds,scoring="neg_mean_squared_error",n_jobs= 10, verbose = 1)
 grid_search.fit(X1,y)
 rid_best=grid_search.best_estimator_
np.sqrt(-grid_search.best_score_ )
train = pd.concat([train,train_orig[['target']]],axis=1)
 train.head()<set_options>
model=Lasso()
las_param_grid = {"alpha":[0.0005963623316594642]}
grid_search= GridSearchCV(model,param_grid=las_param_grid,cv=kfolds,scoring="neg_mean_squared_error",n_jobs= 10, verbose = 1)
 grid_search.fit(X1,y)
 las_best=grid_search.best_estimator_
np.sqrt(-grid_search.best_score_ )
del(train_orig)
 gc.collect()<feature_engineering>
model=ElasticNet()
ela_param_grid = {"alpha":[0.0006951927961775605],
"l1_ratio":[0.90]}
grid_search= GridSearchCV(model,param_grid=ela_param_grid,cv=kfolds,scoring="neg_mean_squared_error",n_jobs= 10, verbose = 1)
 grid_search.fit(X1,y)
 ela_best=grid_search.best_estimator_
np.sqrt(-grid_search.best_score_ )
train['target'] = np.where(train['target'] >= 0.5, True, False )<split>
model=SVR()
svr_param_grid = {"C":[66],
"gamma":[6.105402296585326e-05]}
grid_search= GridSearchCV(model,param_grid=svr_param_grid,cv=kfolds,scoring="neg_mean_squared_error",n_jobs= 10, verbose = 1)
 grid_search.fit(X1,y)
 svr_best=grid_search.best_estimator_
np.sqrt(-grid_search.best_score_ )
train_df, validate_df = model_selection.train_test_split(train, test_size=0.1)
 print('%d train comments, %d validate comments' %(len(train_df), len(validate_df)) )<string_transform>
grid_search.best_params_
MAX_NUM_WORDS = 100000
TOXICITY_COLUMN = 'target'
TEXT_COLUMN = 'comment_text'
tokenizer = Tokenizer(num_words=MAX_NUM_WORDS)
 tokenizer.fit_on_texts(train_df[TEXT_COLUMN])
 MAX_SEQUENCE_LENGTH = 256
def pad_text(texts, tokenizer):
return pad_sequences(tokenizer.texts_to_sequences(texts), maxlen=MAX_SEQUENCE_LENGTH )<set_options>
model=GradientBoostingRegressor()
gbdt_param_grid = {"n_estimators":[2200],
"learning_rate":[0.05],
"max_depth":[3],
"max_features":["sqrt"],
"min_samples_leaf":[5],
"min_samples_split":[12],
"loss":["huber"]
}
grid_search= GridSearchCV(model,param_grid=gbdt_param_grid,cv=kfolds,scoring="neg_mean_squared_error",n_jobs= 10, verbose = 1)
 grid_search.fit(X1,y)
 gbdt_best=grid_search.best_estimator_
np.sqrt(-grid_search.best_score_ )
gc.collect()<define_variables>
model=LGBMRegressor()
lgbm_param_grid = {
'objective':['regression'],
'max_depth':[5],
'num_leaves':[12],
'learning_rate':[0.005],
'n_estimators':[5500],
'max_bin':[190],
'bagging_fraction':[0.2],
'feature_fraction':[0.2]
}
grid_search= GridSearchCV(model,param_grid=lgbm_param_grid,cv=kfolds,scoring="neg_mean_squared_error",n_jobs= 10, verbose = 1)
 grid_search.fit(X1,y)
 lgbm_best=grid_search.best_estimator_
np.sqrt(-grid_search.best_score_ )
EMBEDDINGS_DIMENSION = 300
embedding_matrix = np.zeros(( len(tokenizer.word_index)+ 1,EMBEDDINGS_DIMENSION))<feature_engineering>
kfolds=KFold(n_splits=5,shuffle=True,random_state=42)
 scale=RobustScaler().fit(X)
 X1=scale.transform(X)
 model=XGBRegressor()
xgb_param_grid = {"n_estimators":[3000],
"learning_rate":[0.01],
"max_depth":[3],
"subsample":[0.8],
"colsample_bytree":[0.8],
"gamma":[0],
"objective":['reg:linear'],
"min_child_weight":[2],
"reg_alpha":[0.1],
"reg_lambda":[0.5]
}
grid_search= GridSearchCV(model,param_grid=xgb_param_grid,cv=kfolds,scoring="neg_mean_squared_error",n_jobs= 10, verbose = 1)
 grid_search.fit(X1,y)
 xgb_best=grid_search.best_estimator_
np.sqrt(-grid_search.best_score_ )
num_words_in_embedding = 0
for word, i in tokenizer.word_index.items() :
if word in embeddings_index.vocab:
embedding_vector = embeddings_index[word]
embedding_matrix[i] = embedding_vector
num_words_in_embedding += 1<categorify>
stack_gen = StackingCVRegressor(regressors=(rid_best, las_best, ela_best,
gbdt_best, xgb_best, lgbm_best),
meta_regressor=xgb_best,
use_features_in_secondary=True)
 stack_gen.fit(np.array(X1), np.array(y))
train_text = pad_text(train_df[TEXT_COLUMN], tokenizer)
 train_labels = train_df[TOXICITY_COLUMN]
validate_text = pad_text(validate_df[TEXT_COLUMN], tokenizer)
 validate_labels = validate_df[TOXICITY_COLUMN]<set_options>
X_sub=scale.transform(X_sub )
gc.collect()<normalization>
def blend_models_predict(X):
return(( 0.1 * ela_best.predict(X)) + \ 
(0.1 * las_best.predict(X)) + \ 
(0.1 * rid_best.predict(X)) + \ 
(0.1 * svr_best.predict(X)) + \ 
(0.1 * gbdt_best.predict(X)) + \ 
(0.1 * xgb_best.predict(X)) + \ 
(0.1 * lgbm_best.predict(X)) + \ 
(0.3 * stack_gen.predict(np.array(X))))
print('RMSLE score on train data:')
 print(rmsle(y, blend_models_predict(X1)))
 print('Predict submission', datetime.now() ,)
 submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blend_models_predict(X_sub)))
<choose_model_class><EOS> 
sub_1 = pd.read_csv('.. /input/top-10-0-10943-stacking-mice-and-brutal-force/House_Prices_submit.csv')
 sub_2 = pd.read_csv('.. /input/hybrid-svm-benchmark-approach-0-11180-lb-top-2/hybrid_solution.csv')
 sub_3 = pd.read_csv('.. /input/lasso-model-for-regression-problem/lasso_sol22_Median.csv')
 submission.iloc[:,1] = np.floor(( 0.25 * np.floor(np.expm1(blend_models_predict(X_sub)))) + 
(0.25 * sub_1.iloc[:,1])+ 
(0.25 * sub_2.iloc[:,1])+ 
(0.25 * sub_3.iloc[:,1]))
submission.to_csv("submission.csv", index=False )