Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<train_model>
warnings.filterwarnings(action='ignore', category=DataConversionWarning)
 warnings.filterwarnings(action='ignore', category=DeprecationWarning)
 warnings.filterwarnings(action='ignore', category=FutureWarning )
def runMNB(train_X, train_y, test_X, test_y, test_X2):
model = naive_bayes.MultinomialNB()
model.fit(train_X, train_y)
 pred_test_y = model.predict_proba(test_X)
 pred_test_y2 = model.predict_proba(test_X2)
 return pred_test_y, pred_test_y2, model<compute_train_metric>
train = pd.read_csv(".. /input/train.csv")
 compe = pd.read_csv(".. /input/test.csv")
 sample_sub = pd.read_csv(".. /input/sample_submission.csv")
 data = train.append(compe, sort=False )
cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("Mean cv score : ", np.mean(cv_scores))
pred_full_test = pred_full_test / 5 .<concatenate>
types = pd.DataFrame(data.dtypes ).rename(columns={0: 'type'} ).sort_values(by=['type'],ascending=False)
 types
n_comp = 20
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_word_'+str(i)for i in range(n_comp)]
test_svd.columns = ['svd_word_'+str(i)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd<feature_engineering>
def check_missing(df):
null_val = df.isnull().sum()
percent = 100 * df.isnull().sum() /len(df)
 missing_table = pd.concat([null_val, percent], axis=1)
 col = missing_table.rename(columns = {0 : 'Num', 1 : 'Rate'})
 return col
print("Data
cols = check_missing(data)
 types.join(cols ).sort_values(by="Rate", ascending=False )
tfidf_vec = CountVectorizer(stop_words='english', ngram_range=(1,3))
tfidf_vec.fit(train_df['text'].values.tolist() + test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist() )<compute_train_metric>
data.drop(['PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu'], axis=1, inplace = True )
cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("Mean cv score : ", np.mean(cv_scores))
pred_full_test = pred_full_test / 5.
 train_df["nb_cvec_eap"] = pred_train[:,0]
train_df["nb_cvec_hpl"] = pred_train[:,1]
train_df["nb_cvec_mws"] = pred_train[:,2]
test_df["nb_cvec_eap"] = pred_full_test[:,0]
test_df["nb_cvec_hpl"] = pred_full_test[:,1]
test_df["nb_cvec_mws"] = pred_full_test[:,2]<categorify>
categorical_variables_string = \
['MSZoning', 'Street', 'LotShape', 'LandContour',
'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood',
'Condition1', 'Condition2', 'BldgType', 'HouseStyle',
'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd',
'MasVnrType', 'ExterQual', 'ExterCond', 'Foundation',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'Heating', 'HeatingQC', 'CentralAir',
'Electrical', 'KitchenQual', 'Functional', 'GarageType',
'GarageFinish', 'GarageQual', 'GarageCond', 'PavedDrive',
'SaleType', 'SaleCondition']
for v in categorical_variables_string:
data[v] = data[v].fillna(data[v].mode() [0])
 data = pd.get_dummies(data, columns=[v], drop_first=True)
tfidf_vec = CountVectorizer(ngram_range=(1,7), analyzer='char')
 tfidf_vec.fit(train_df['text'].values.tolist() + test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist())
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("Mean cv score : ", np.mean(cv_scores))
pred_full_test = pred_full_test / 5.
 train_df["nb_cvec_char_eap"] = pred_train[:,0]
train_df["nb_cvec_char_hpl"] = pred_train[:,1]
train_df["nb_cvec_char_mws"] = pred_train[:,2]
test_df["nb_cvec_char_eap"] = pred_full_test[:,0]
test_df["nb_cvec_char_hpl"] = pred_full_test[:,1]
test_df["nb_cvec_char_mws"] = pred_full_test[:,2]<categorify>
categorical_variables_int = \
['OverallQual', 'OverallCond', 'MoSold']
tfidf_vec = TfidfVectorizer(ngram_range=(1,5), analyzer='char')
 full_tfidf = tfidf_vec.fit_transform(train_df['text'].values.tolist() + test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist())
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("Mean cv score : ", np.mean(cv_scores))
pred_full_test = pred_full_test / 5.
 train_df["nb_tfidf_char_eap"] = pred_train[:,0]
train_df["nb_tfidf_char_hpl"] = pred_train[:,1]
train_df["nb_tfidf_char_mws"] = pred_train[:,2]
test_df["nb_tfidf_char_eap"] = pred_full_test[:,0]
test_df["nb_tfidf_char_hpl"] = pred_full_test[:,1]
test_df["nb_tfidf_char_mws"] = pred_full_test[:,2]<concatenate>
numerical_variavles = \
['MSSubClass', 'LotFrontage', 'LotArea', 'YearBuilt',
'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2',
'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF',
'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath',
'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr',
'TotRmsAbvGrd', 'Fireplaces', 'GarageYrBlt', 'GarageCars',
'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch',
'3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal', 'YrSold']
ss = StandardScaler()
for v in numerical_variavles:
data[v] = data[v].fillna(data[v].mean())
 data[v] = ss.fit_transform(data[[v]] )
n_comp = 20
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_char_'+str(i)for i in range(n_comp)]
test_svd.columns = ['svd_char_'+str(i)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd<split>
train = data[:1460]
test = data[1460:]
cols_to_drop = ['id', 'text']
train_X = train_df.drop(cols_to_drop+['author'], axis=1)
 test_X = test_df.drop(cols_to_drop, axis=1)
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_X.loc[dev_index], train_X.loc[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runXGB(dev_X, dev_y, val_X, val_y, test_X, seed_val=0, colsample=0.7)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
break
print("cv scores : ", cv_scores)
 out_df = pd.DataFrame(pred_full_test)
 out_df.columns = ['EAP', 'HPL', 'MWS']
out_df.insert(0, 'id', test_id)
 out_df.to_csv("sub_fe.csv", index=False )<import_modules>
possible_features = train.columns.copy().drop('SalePrice' ).drop('Id')
 selector = SelectKBest(f_regression, len(possible_features))
selector.fit(train[possible_features], train['SalePrice'])
 scores = -np.log10(selector.pvalues_)
 indices = np.argsort(scores)[::-1]
print('Feature importances:')
 for i in range(len(scores)) :
print('%.2f %s' %(scores[indices[i]], possible_features[indices[i]]))
import af<import_modules>
fparams = \
['OverallQual', 'GrLivArea', 'GarageCars', 'GarageArea', 'TotalBsmtSF',
'1stFlrSF', 'ExterQual_TA', 'FullBath', 'TotRmsAbvGrd', 'YearBuilt',
'KitchenQual_TA', 'GarageFinish_Unf', 'YearRemodAdd', 'BsmtQual_TA',
'Foundation_PConc', 'MasVnrArea', 'GarageYrBlt', 'Fireplaces',
'ExterQual_Gd', 'BsmtFinType1_GLQ', 'Neighborhood_NridgHt', 'BsmtFinSF1',
'MasVnrType_None', 'SaleType_New', 'GarageType_Detchd', 'SaleCondition_Partial',
'Foundation_CBlock', 'LotFrontage', 'MasVnrType_Stone', 'Neighborhood_NoRidge',
'WoodDeckSF', 'KitchenQual_Gd', 'BsmtExposure_No', '2ndFlrSF', 'OpenPorchSF',
'HeatingQC_TA', 'BsmtExposure_Gd', 'Exterior2nd_VinylSd', 'Exterior1st_VinylSd',
'MSZoning_RM', 'HalfBath', 'LotShape_Reg', 'LotArea', 'CentralAir_Y', 'MSZoning_RL',
'HouseStyle_2Story', 'SaleType_WD', 'Electrical_SBrkr', 'RoofStyle_Hip', 'GarageType_BuiltIn',
'BsmtQual_Gd', 'GarageType_Attchd', 'PavedDrive_Y', 'BsmtFullBath', 'RoofStyle_Gable',
'Neighborhood_StoneBr', 'BsmtUnfSF', 'MasVnrType_BrkFace', 'Neighborhood_OldTown',
'Neighborhood_NAmes', 'Neighborhood_Edwards', 'GarageFinish_RFn', 'RoofMatl_WdShngl',
'BedroomAbvGr', 'Exterior1st_MetalSd', 'Neighborhood_IDOTRR', 'Exterior2nd_MetalSd',
'Exterior2nd_Wd Sdng', 'Exterior1st_Wd Sdng', 'KitchenQual_Fa', 'SaleCondition_Normal',
'Neighborhood_BrkSide', 'LotConfig_CulDSac', 'Neighborhood_Somerst', 'ExterCond_Fa',
'GarageCond_TA', 'KitchenAbvGr', 'BsmtFinType1_Rec', 'HeatingQC_Gd', 'HeatingQC_Fa',
'Exterior1st_CemntBd', 'GarageQual_Fa', 'BsmtFinType1_Unf', 'BsmtFinType1_BLQ',
'GarageCond_Fa', 'BsmtQual_Fa', 'EnclosedPorch', 'Neighborhood_Sawyer', 'Exterior2nd_CmentBd',
'Electrical_FuseF', 'Neighborhood_Timber', 'LotShape_IR2', 'LandContour_HLS', 'Foundation_Slab',
'Condition1_Feedr', 'Functional_Typ', 'ExterQual_Fa', 'BldgType_Duplex', 'Condition1_Norm',
'Neighborhood_MeadowV', 'ScreenPorch', 'ExterCond_TA', 'RoofMatl_CompShg', 'Neighborhood_BrDale',
'BldgType_Twnhs', 'BldgType_2fmCon', 'GarageQual_TA', 'Exterior1st_HdBoard', 'HouseStyle_SFoyer',
'Heating_GasA', 'PoolArea', 'Heating_Grav', 'MSZoning_FV', 'BsmtCond_Gd', 'PavedDrive_P',
'HouseStyle_1.5Unf', 'BsmtFinType1_LwQ', 'MSSubClass', 'LotConfig_Inside', 'OverallCond',
'Exterior2nd_ImStucc', 'Neighborhood_CollgCr', 'Functional_Min2', 'Neighborhood_Crawfor',
'GarageType_CarPort', 'Functional_Maj2', 'Exterior2nd_HdBoard']
train_target = train["SalePrice"].values
train_features = train[fparams].values
test_features = test[fparams].values
alpha_tokenizer = RegexpTokenizer('[A-Za-z]\w+')
 lemmatizer = WordNetLemmatizer()
stop = stopwords.words('english' )<load_from_csv>
svrgs_parameters = {
'kernel': ['rbf'],
'C': [150000,200000,250000],
'gamma': [0.004,0.0045,0.005]
}
svr_cv = GridSearchCV(svm.SVR() , svrgs_parameters, cv=8, scoring= 'neg_mean_squared_log_error')
 svr_cv.fit(train_features, train_target)
 print("SVR GridSearch score: "+str(svr_cv.best_score_))
print("SVR GridSearch params: ")
 print(svr_cv.best_params_ )
<feature_engineering><EOS> 
prediction = svr_cv.best_estimator_.predict(test_features)
 pred = pd.DataFrame(pd.read_csv(".. /input/test.csv")['Id'])
 pred['SalePrice'] = prediction
pred.to_csv(".. /working/submission.csv", index = False )